Example:
simple flat
nested objects
lists of primitives
sparse data
coingecko coins
complex mixed
simple_flat.json — Original size: 763 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 763tokens: 384chars_og/tok: 2.0encoded_chars/tok: 2.0
[↵
· {↵
··· "id": 1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id": 2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id": 3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id": 4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
· {↵
··· "id": 1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id": 2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id": 3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id": 4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
JSON (min)
chars: 522tokens: 152chars_og/tok: 5.0encoded_chars/tok: 3.4
[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 95chars_og/tok: 8.0encoded_chars/tok: 2.5
id,name,role,department
↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
TSV
chars: 234tokens: 95chars_og/tok: 8.0encoded_chars/tok: 2.5
id→name→role→department
↵
1→Alice→Engineer→Backend ↵
2→Bob→Designer→Frontend ↵
3→Carol→Manager→Product ↵
4→David→Engineer→Infrastructure ↵
5→Eva→Analyst→Data ↵
6→Frank→Engineer→Backend ↵
7→Grace→Designer→Mobile ↵
8→Henry→Manager→Engineering ↵
1→Alice→Engineer→Backend ↵
2→Bob→Designer→Frontend ↵
3→Carol→Manager→Product ↵
4→David→Engineer→Infrastructure ↵
5→Eva→Analyst→Data ↵
6→Frank→Engineer→Backend ↵
7→Grace→Designer→Mobile ↵
8→Henry→Manager→Engineering ↵
YAML
chars: 489tokens: 163chars_og/tok: 4.7encoded_chars/tok: 3.0
- department: Backend↵
· id: 1↵
· name: Alice↵
· role: Engineer↵
- department: Frontend↵
· id: 2↵
· name: Bob↵
· role: Designer↵
- department: Product↵
· id: 3↵
· name: Carol↵
· role: Manager↵
- department: Infrastructure↵
· id: 4↵
· name: David↵
· role: Engineer↵
- department: Data↵
· id: 5↵
· name: Eva↵
· role: Analyst↵
- department: Backend↵
· id: 6↵
· name: Frank↵
· role: Engineer↵
- department: Mobile
... (truncated)
· id: 1↵
· name: Alice↵
· role: Engineer↵
- department: Frontend↵
· id: 2↵
· name: Bob↵
· role: Designer↵
- department: Product↵
· id: 3↵
· name: Carol↵
· role: Manager↵
- department: Infrastructure↵
· id: 4↵
· name: David↵
· role: Engineer↵
- department: Data↵
· id: 5↵
· name: Eva↵
· role: Analyst↵
- department: Backend↵
· id: 6↵
· name: Frank↵
· role: Engineer↵
- department: Mobile
... (truncated)
TOON
chars: 246tokens: 98chars_og/tok: 7.8encoded_chars/tok: 2.5
[8]{id,name,role,department}:↵
· 1,Alice,Engineer,Backend↵
· 2,Bob,Designer,Frontend↵
· 3,Carol,Manager,Product↵
· 4,David,Engineer,Infrastructure↵
· 5,Eva,Analyst,Data↵
· 6,Frank,Engineer,Backend↵
· 7,Grace,Designer,Mobile↵
· 8,Henry,Manager,Engineering
· 1,Alice,Engineer,Backend↵
· 2,Bob,Designer,Frontend↵
· 3,Carol,Manager,Product↵
· 4,David,Engineer,Infrastructure↵
· 5,Eva,Analyst,Data↵
· 6,Frank,Engineer,Backend↵
· 7,Grace,Designer,Mobile↵
· 8,Henry,Manager,Engineering
TSON
chars: 229tokens: 90chars_og/tok: 8.5encoded_chars/tok: 2.5
{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 74chars_og/tok: 10.3encoded_chars/tok: 3.4
id; name; role; department↵
1; Alice; Engineer; Backend↵
2; Bob; Designer; Frontend↵
3; Carol; Manager; Product↵
4; David; Engineer; Infrastructure↵
5; Eva; Analyst; Data↵
6; Frank; Engineer; Backend↵
7; Grace; Designer; Mobile↵
8; Henry; Manager; Engineering
1; Alice; Engineer; Backend↵
2; Bob; Designer; Frontend↵
3; Carol; Manager; Product↵
4; David; Engineer; Infrastructure↵
5; Eva; Analyst; Data↵
6; Frank; Engineer; Backend↵
7; Grace; Designer; Mobile↵
8; Henry; Manager; Engineering
minemizer (compact)
chars: 224tokens: 85chars_og/tok: 9.0encoded_chars/tok: 2.6
id;name;role;department↵
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 741tokens: 407chars_og/tok: 1.8encoded_chars/tok: 1.8
[↵
· {↵
··· "id": 1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id": 2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id": 3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
· {↵
··· "id": 1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id": 2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id": 3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
JSON (min)
chars: 470tokens: 143chars_og/tok: 5.2encoded_chars/tok: 3.3
[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 195chars_og/tok: 3.8encoded_chars/tok: 2.4
- id: 1↵
· status: active↵
· user:↵
··· email: alice@example.com↵
··· name: Alice↵
- id: 2↵
· status: inactive↵
· user:↵
··· email: bob@example.com↵
··· name: Bob↵
- id: 3↵
· status: active↵
· user:↵
··· email: carol@example.com↵
··· name: Carol↵
- id: 4↵
· status: pending↵
· user:↵
··· email: david@example.com↵
··· name: David↵
- id: 5↵
· status: active↵
· user:↵
··· email: eva@example.com↵
··· name: Eva
... (truncated)
· status: active↵
· user:↵
··· email: alice@example.com↵
··· name: Alice↵
- id: 2↵
· status: inactive↵
· user:↵
··· email: bob@example.com↵
··· name: Bob↵
- id: 3↵
· status: active↵
· user:↵
··· email: carol@example.com↵
··· name: Carol↵
- id: 4↵
· status: pending↵
· user:↵
··· email: david@example.com↵
··· name: David↵
- id: 5↵
· status: active↵
· user:↵
··· email: eva@example.com↵
··· name: Eva
... (truncated)
TOON
chars: 527tokens: 252chars_og/tok: 2.9encoded_chars/tok: 2.1
[6]:↵
· - id: 1↵
··· user:↵
····· name: Alice↵
····· email: alice@example.com↵
··· status: active↵
· - id: 2↵
··· user:↵
····· name: Bob↵
····· email: bob@example.com↵
··· status: inactive↵
· - id: 3↵
··· user:↵
····· name: Carol↵
····· email: carol@example.com↵
··· status: active↵
· - id: 4↵
··· user:↵
····· name: David↵
····· email: david@example.com↵
··· status: pending↵
· - id: 5↵
··· user:↵
····· name: Eva↵
····· email: eva@example.com
... (truncated)
· - id: 1↵
··· user:↵
····· name: Alice↵
····· email: alice@example.com↵
··· status: active↵
· - id: 2↵
··· user:↵
····· name: Bob↵
····· email: bob@example.com↵
··· status: inactive↵
· - id: 3↵
··· user:↵
····· name: Carol↵
····· email: carol@example.com↵
··· status: active↵
· - id: 4↵
··· user:↵
····· name: David↵
····· email: david@example.com↵
··· status: pending↵
· - id: 5↵
··· user:↵
····· name: Eva↵
····· email: eva@example.com
... (truncated)
TSON
chars: 249tokens: 101chars_og/tok: 7.3encoded_chars/tok: 2.5
{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 90chars_og/tok: 8.2encoded_chars/tok: 2.9
id; user{ name; email}; status↵
1; { Alice; alice@example.com}; active↵
2; { Bob; bob@example.com}; inactive↵
3; { Carol; carol@example.com}; active↵
4; { David; david@example.com}; pending↵
5; { Eva; eva@example.com}; active↵
6; { Frank; frank@example.com}; active
1; { Alice; alice@example.com}; active↵
2; { Bob; bob@example.com}; inactive↵
3; { Carol; carol@example.com}; active↵
4; { David; david@example.com}; pending↵
5; { Eva; eva@example.com}; active↵
6; { Frank; frank@example.com}; active
minemizer (compact)
chars: 232tokens: 95chars_og/tok: 7.8encoded_chars/tok: 2.4
id;user{ name;email};status↵
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 610tokens: 382chars_og/tok: 1.6encoded_chars/tok: 1.6
[↵
· {↵
··· "id": 1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id": 2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id": 3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
· {↵
··· "id": 1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id": 2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id": 3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
JSON (min)
chars: 330tokens: 115chars_og/tok: 5.3encoded_chars/tok: 2.9
[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 153chars_og/tok: 4.0encoded_chars/tok: 2.2
- id: 1↵
· name: Alice↵
· skills:↵
· - python↵
· - go↵
· - rust↵
- id: 2↵
· name: Bob↵
· skills:↵
· - javascript↵
· - typescript↵
- id: 3↵
· name: Carol↵
· skills:↵
· - java↵
· - kotlin↵
· - scala↵
· - groovy↵
- id: 4↵
· name: David↵
· skills:↵
· - c↵
· - cpp↵
- id: 5↵
· name: Eva
... (truncated)
· name: Alice↵
· skills:↵
· - python↵
· - go↵
· - rust↵
- id: 2↵
· name: Bob↵
· skills:↵
· - javascript↵
· - typescript↵
- id: 3↵
· name: Carol↵
· skills:↵
· - java↵
· - kotlin↵
· - scala↵
· - groovy↵
- id: 4↵
· name: David↵
· skills:↵
· - c↵
· - cpp↵
- id: 5↵
· name: Eva
... (truncated)
TOON
chars: 339tokens: 161chars_og/tok: 3.8encoded_chars/tok: 2.1
[6]:↵
· - id: 1↵
··· name: Alice↵
··· skills[3]: python,go,rust↵
· - id: 2↵
··· name: Bob↵
··· skills[2]: javascript,typescript↵
· - id: 3↵
··· name: Carol↵
··· skills[4]: java,kotlin,scala,groovy↵
· - id: 4↵
··· name: David↵
··· skills[2]: c,cpp↵
· - id: 5↵
··· name: Eva↵
··· skills[3]: ruby,elixir,erlang↵
· - id: 6↵
··· name: Frank↵
··· skills[1]: swift
· - id: 1↵
··· name: Alice↵
··· skills[3]: python,go,rust↵
· - id: 2↵
··· name: Bob↵
··· skills[2]: javascript,typescript↵
· - id: 3↵
··· name: Carol↵
··· skills[4]: java,kotlin,scala,groovy↵
· - id: 4↵
··· name: David↵
··· skills[2]: c,cpp↵
· - id: 5↵
··· name: Eva↵
··· skills[3]: ruby,elixir,erlang↵
· - id: 6↵
··· name: Frank↵
··· skills[1]: swift
TSON
chars: 168tokens: 80chars_og/tok: 7.6encoded_chars/tok: 2.1
{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 81chars_og/tok: 7.5encoded_chars/tok: 2.4
id; name; skills[]↵
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
minemizer (compact)
chars: 165tokens: 83chars_og/tok: 7.3encoded_chars/tok: 2.0
id;name;skills[]↵
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 589tokens: 318chars_og/tok: 1.9encoded_chars/tok: 1.9
[↵
· {↵
··· "id": 1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id": 2,↵
··· "name": "Dave",↵
··· "remote": true↵
· },↵
· {↵
··· "id": 3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id": 4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id": 5,↵
··· "name": "Grace",
... (truncated)
· {↵
··· "id": 1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id": 2,↵
··· "name": "Dave",↵
··· "remote": true↵
· },↵
· {↵
··· "id": 3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id": 4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id": 5,↵
··· "name": "Grace",
... (truncated)
JSON (min)
chars: 378tokens: 121chars_og/tok: 4.9encoded_chars/tok: 3.1
[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 135chars_og/tok: 4.4encoded_chars/tok: 2.6
- id: 1↵
· name: Carol↵
· role: Manager↵
- id: 2↵
· name: Dave↵
· remote: true↵
- id: 3↵
· name: Eve↵
· role: Designer↵
· team: UX↵
- department: Engineering↵
· id: 4↵
· name: Frank↵
- id: 5↵
· name: Grace↵
· remote: true↵
· role: Engineer↵
· team: Platform↵
- id: 6↵
· name: Henry↵
· role: Analyst↵
- id: 7↵
· name: Ivy↵
- department: Sales↵
· id: 8
... (truncated)
· name: Carol↵
· role: Manager↵
- id: 2↵
· name: Dave↵
· remote: true↵
- id: 3↵
· name: Eve↵
· role: Designer↵
· team: UX↵
- department: Engineering↵
· id: 4↵
· name: Frank↵
- id: 5↵
· name: Grace↵
· remote: true↵
· role: Engineer↵
· team: Platform↵
- id: 6↵
· name: Henry↵
· role: Analyst↵
- id: 7↵
· name: Ivy↵
- department: Sales↵
· id: 8
... (truncated)
TOON
chars: 414tokens: 184chars_og/tok: 3.2encoded_chars/tok: 2.2
[8]:↵
· - id: 1↵
··· name: Carol↵
··· role: Manager↵
· - id: 2↵
··· name: Dave↵
··· remote: true↵
· - id: 3↵
··· name: Eve↵
··· role: Designer↵
··· team: UX↵
· - id: 4↵
··· name: Frank↵
··· department: Engineering↵
· - id: 5↵
··· name: Grace↵
··· role: Engineer↵
··· remote: true↵
··· team: Platform↵
· - id: 6↵
··· name: Henry↵
··· role: Analyst↵
· - id: 7↵
··· name: Ivy↵
· - id: 8
... (truncated)
· - id: 1↵
··· name: Carol↵
··· role: Manager↵
· - id: 2↵
··· name: Dave↵
··· remote: true↵
· - id: 3↵
··· name: Eve↵
··· role: Designer↵
··· team: UX↵
· - id: 4↵
··· name: Frank↵
··· department: Engineering↵
· - id: 5↵
··· name: Grace↵
··· role: Engineer↵
··· remote: true↵
··· team: Platform↵
· - id: 6↵
··· name: Henry↵
··· role: Analyst↵
· - id: 7↵
··· name: Ivy↵
· - id: 8
... (truncated)
TSON
chars: 300tokens: 136chars_og/tok: 4.3encoded_chars/tok: 2.2
[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 79chars_og/tok: 7.5encoded_chars/tok: 2.9
id; name; role↵
1; Carol; Manager↵
2; Dave; ; remote:True↵
3; Eve; Designer; team:UX↵
4; Frank; ; department:Engineering↵
5; Grace; Engineer; remote:True; team:Platform↵
6; Henry; Analyst↵
7; Ivy;·↵
8; Jack; ; department:Sales; remote:False
1; Carol; Manager↵
2; Dave; ; remote:True↵
3; Eve; Designer; team:UX↵
4; Frank; ; department:Engineering↵
5; Grace; Engineer; remote:True; team:Platform↵
6; Henry; Analyst↵
7; Ivy;·↵
8; Jack; ; department:Sales; remote:False
minemizer (compact)
chars: 207tokens: 84chars_og/tok: 7.0encoded_chars/tok: 2.5
id;name;role↵
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 1,611,780tokens: 862,469chars_og/tok: 1.9encoded_chars/tok: 1.9
[↵
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 400,135chars_og/tok: 4.0encoded_chars/tok: 2.9
[{"id":"_","symbol":"gib","name":"\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000 Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIX Protocol"},{"id":"0x","symbol":"zrx","name":"0x Protocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai: AI Smart Contract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0x Leverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonk by Virtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"Avalanche Bridged ZRX (Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsim by Virtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS (Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000x by Virtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10 figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1 Coin Can Change Your Life"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1 community can change your life"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1 dog can change your life"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1Hive Water"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCH yVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1Move Token"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1 narrative can change your life"},{"id":"1-one","symbol":"one","name":"1 (one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUS DAO"},{"id":"1-squirrel","symbol":"peanut","name":"OG Peanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004 PEPE"},{"id":"2025-token","symbol":"2025","name":"2025 TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077 CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23 Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3 years and forget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24K Gold PEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2G Carbon Coin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"Meow Meow Coin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACE ART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
... (truncated)
CSV
chars: 603,635tokens: 282,641chars_og/tok: 5.7encoded_chars/tok: 2.1
id,symbol,name
↵
_,gib,�� �� ��_��·���� ↵
000-capital,000,000 Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIX Protocol ↵
0x,zrx,0x Protocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0x Leverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsim by Virtuals ↵
0xy,0xy,0xy ↵
-10,loong,�� ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
_,gib,�� �� ��_��·���� ↵
000-capital,000,000 Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIX Protocol ↵
0x,zrx,0x Protocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0x Leverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsim by Virtuals ↵
0xy,0xy,0xy ↵
-10,loong,�� ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 282,672chars_og/tok: 5.7encoded_chars/tok: 2.1
id→symbol→name
↵
_→gib→�� �� ��_��·���� ↵
000-capital→000→000 Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain→zcn→Zus ↵
0vix-protocol→vix→0VIX Protocol ↵
0x→zrx→0x Protocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933→wolf→Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen→xgn→0xGen ↵
0x-leverage→oxl→0x Leverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e→zrx.e→Avalanche Bridged ZRX (Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals→sage→0xsim by Virtuals ↵
0xy→0xy→0xy ↵
-10→loong→�� ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
_→gib→�� �� ��_��·���� ↵
000-capital→000→000 Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain→zcn→Zus ↵
0vix-protocol→vix→0VIX Protocol ↵
0x→zrx→0x Protocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933→wolf→Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen→xgn→0xGen ↵
0x-leverage→oxl→0x Leverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e→zrx.e→Avalanche Bridged ZRX (Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals→sage→0xsim by Virtuals ↵
0xy→0xy→0xy ↵
-10→loong→�� ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 429,115chars_og/tok: 3.8encoded_chars/tok: 2.4
- id: _↵
· name:·�� �� ��_��·����↵
· symbol: gib↵
- id: 000-capital↵
· name: 000 Capital↵
· symbol: '000'↵
- id: 01111010011110000110001001110100-token↵
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id: 01-token↵
· name: '01'↵
· symbol: '01'↵
- id: 0chain↵
· name: Zus↵
· symbol: zcn↵
- id: 0vix-protocol↵
· name: 0VIX Protocol↵
· symbol: vix↵
- id: 0x↵
· name: 0x Protocol↵
· symbol: zrx↵
- id: 0x0-ai-ai-smart-contract↵
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id: 0x678-landwolf-1933
... (truncated)
· name:·�� �� ��_��·����↵
· symbol: gib↵
- id: 000-capital↵
· name: 000 Capital↵
· symbol: '000'↵
- id: 01111010011110000110001001110100-token↵
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id: 01-token↵
· name: '01'↵
· symbol: '01'↵
- id: 0chain↵
· name: Zus↵
· symbol: zcn↵
- id: 0vix-protocol↵
· name: 0VIX Protocol↵
· symbol: vix↵
- id: 0x↵
· name: 0x Protocol↵
· symbol: zrx↵
- id: 0x0-ai-ai-smart-contract↵
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id: 0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 279,372chars_og/tok: 5.8encoded_chars/tok: 2.2
[19332]{id,symbol,name}:↵
· _,gib,�� �� ��_��·����↵
· 000-capital,"000",000 Capital↵
· 01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
· 01-token,"01","01"↵
· 0chain,zcn,Zus↵
· 0vix-protocol,vix,0VIX Protocol↵
· 0x,zrx,0x Protocol↵
· 0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
· 0x678-landwolf-1933,wolf,Landwolf↵
· 0xgasless-2,0xgas,0xGasless↵
· 0xgen,xgn,0xGen↵
· 0x-leverage,oxl,0x Leverage↵
· 0xlsd,0xlsd,0xLSD↵
· 0xmonk,monk,0xMonk by Virtuals↵
· 0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
· 0xshadow,0xs,0xShadow↵
· 0xsim-by-virtuals,sage,0xsim by Virtuals↵
· 0xy,0xy,0xy↵
· "-10",loong,��↵
· 1000bonk,1000bonk,1000BONK↵
· 1000btt,1000btt,1000BTT↵
· 1000cat,1000cat,1000CAT↵
· 1000chems,1000cheems,1000CHEMS↵
· 1000mog,1000mog,1000MOG
... (truncated)
· _,gib,�� �� ��_��·����↵
· 000-capital,"000",000 Capital↵
· 01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
· 01-token,"01","01"↵
· 0chain,zcn,Zus↵
· 0vix-protocol,vix,0VIX Protocol↵
· 0x,zrx,0x Protocol↵
· 0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
· 0x678-landwolf-1933,wolf,Landwolf↵
· 0xgasless-2,0xgas,0xGasless↵
· 0xgen,xgn,0xGen↵
· 0x-leverage,oxl,0x Leverage↵
· 0xlsd,0xlsd,0xLSD↵
· 0xmonk,monk,0xMonk by Virtuals↵
· 0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
· 0xshadow,0xs,0xShadow↵
· 0xsim-by-virtuals,sage,0xsim by Virtuals↵
· 0xy,0xy,0xy↵
· "-10",loong,��↵
· 1000bonk,1000bonk,1000BONK↵
· 1000btt,1000btt,1000BTT↵
· 1000cat,1000cat,1000CAT↵
· 1000chems,1000cheems,1000CHEMS↵
· 1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 271,823chars_og/tok: 5.9encoded_chars/tok: 2.2
{@id,symbol,name#19332|_,gib,"�� �� ��_��·����"|000-capital,"000","000 Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIX Protocol"|0x,zrx,"0x Protocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0x Leverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonk by Virtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"Avalanche Bridged ZRX (Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsim by Virtuals"|0xy,0xy,0xy|"-10",loong,��|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS (Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000x by Virtuals"|100-token,100��,100��|100xdarren,100x,100xDarren|10-figs,figs,"10 figs"|"-11",���������,���������|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1 Coin Can Change Your Life"|1-community-can-change-your-life,community,"1 community can change your life"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1 dog can change your life"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1Hive Water"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCH yVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1Move Token"|1-narrative-can-change-your-life,narrative,"1 narrative can change your life"|1-one,one,"1 (one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUS DAO"|1-squirrel,peanut,"OG Peanut"|1-token,"1","1"|2004-pepe,bog,"2004 PEPE"|2025-token,"2025","2025 TOKEN"|2077-code,"2077","2077 CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23 Turtles"|2-3-years-and-forget,"23","2-3 years and forget"|24k-gold-pepe,goldpepe,"24K Gold PEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2G Carbon Coin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"Meow Meow Coin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACE ART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404—GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47th POTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4 Next Unicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TB Coin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-Way Mirror Money"|"-5",������,������������������|500m-piece-of-paper,paper,"$500M piece of paper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5th Scape"|"-6","��","��"|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",��,Voidify|717ai-by-virtuals,wire,"717ai by Virtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",���,���|8004-dog,dog8004,"8004 Dog"|8008-token,"8008","8008"|888coin,���������,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8 Ball"|8-bit-coin,coin,"8-Bit Coin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",��,��|9-5,9-5,9to5|99-bitcoins,99btc,"99 Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16z AI Dog"|a51-finance,a51,"A51 Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaa cat"|aaai_agent-by-virtuals,aaai,"AAAI_agent by Virtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADex Finance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"Aark Digital"|aarna-afi-802v2,"afi 802v2","aarna afi 802v2"|aarna-atv111,atv111,"aarna atv111"|aarna-atv111-arbitrum,atv111,"aarna atv111 (Arbitrum)"|aarna-atv111-sonic,atv111,"aarna atv111 (Sonic)"|aarna-atv-808,atv808,"aarna atv 808"|aarna-atv-usdc,atvusdc,"aarna atv USDC (Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarna atv USDC (Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"Aave AAVE"|aave-amm-bptbalweth,aammbptbalweth,"Aave AMM BptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"Aave AMM BptWBTCWETH"|aave-amm-dai,aammdai,"Aave AMM DAI"|aave-amm-uniaaveweth,aammuniaaveweth,"Aave AMM UniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"Aave AMM UniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"Aave AMM UniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"Aave AMM UniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"Aave AMM UniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"Aave AMM UniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"Aave AMM·
... (truncated)
... (truncated)
minemizer
chars: 622,925tokens: 258,209chars_og/tok: 6.2encoded_chars/tok: 2.4
id; symbol; name↵
_; gib;·�� �� ��_��·����↵
000-capital; 000; 000 Capital↵
01111010011110000110001001110100-token; 01111010011110000110001001110100; 01111010011110000110001001110100↵
01-token; 01; 01↵
0chain; zcn; Zus↵
0vix-protocol; vix; 0VIX Protocol↵
0x; zrx; 0x Protocol↵
0x0-ai-ai-smart-contract; 0x0; 0x0.ai: AI Smart Contract↵
0x678-landwolf-1933; wolf; Landwolf↵
0xgasless-2; 0xgas; 0xGasless↵
0xgen; xgn; 0xGen↵
0x-leverage; oxl; 0x Leverage↵
0xlsd; 0xlsd; 0xLSD↵
0xmonk; monk; 0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow; 0xs; 0xShadow↵
0xsim-by-virtuals; sage; 0xsim by Virtuals↵
0xy; 0xy; 0xy↵
-10; loong;·��↵
1000bonk; 1000bonk; 1000BONK↵
1000btt; 1000btt; 1000BTT↵
1000cat; 1000cat; 1000CAT↵
1000chems; 1000cheems; 1000CHEMS↵
1000mog; 1000mog; 1000MOG
... (truncated)
_; gib;·�� �� ��_��·����↵
000-capital; 000; 000 Capital↵
01111010011110000110001001110100-token; 01111010011110000110001001110100; 01111010011110000110001001110100↵
01-token; 01; 01↵
0chain; zcn; Zus↵
0vix-protocol; vix; 0VIX Protocol↵
0x; zrx; 0x Protocol↵
0x0-ai-ai-smart-contract; 0x0; 0x0.ai: AI Smart Contract↵
0x678-landwolf-1933; wolf; Landwolf↵
0xgasless-2; 0xgas; 0xGasless↵
0xgen; xgn; 0xGen↵
0x-leverage; oxl; 0x Leverage↵
0xlsd; 0xlsd; 0xLSD↵
0xmonk; monk; 0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow; 0xs; 0xShadow↵
0xsim-by-virtuals; sage; 0xsim by Virtuals↵
0xy; 0xy; 0xy↵
-10; loong;·��↵
1000bonk; 1000bonk; 1000BONK↵
1000btt; 1000btt; 1000BTT↵
1000cat; 1000cat; 1000CAT↵
1000chems; 1000cheems; 1000CHEMS↵
1000mog; 1000mog; 1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 263,320chars_og/tok: 6.1encoded_chars/tok: 2.2
id;symbol;name↵
_;gib;�� �� ��_��·����↵
000-capital;000;000 Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIX Protocol↵
0x;zrx;0x Protocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0x Leverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsim by Virtuals↵
0xy;0xy;0xy↵
-10;loong;��↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
_;gib;�� �� ��_��·����↵
000-capital;000;000 Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIX Protocol↵
0x;zrx;0x Protocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0x Leverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsim by Virtuals↵
0xy;0xy;0xy↵
-10;loong;��↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: gpt2 (openai-community/gpt2)
JSON (pretty)
chars: 1,320tokens: 768chars_og/tok: 1.7encoded_chars/tok: 1.7
[↵
· {↵
··· "id": 1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id": 2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
· {↵
··· "id": 1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id": 2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
JSON (min)
chars: 760tokens: 224chars_og/tok: 5.9encoded_chars/tok: 3.4
[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 374chars_og/tok: 3.5encoded_chars/tok: 2.2
- id: 1↵
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC↵
····· country: USA↵
··· name: Grace↵
· tags:↵
· - admin↵
· - verified↵
- id: 2↵
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London↵
····· country: UK↵
··· name: Henry↵
· tags:↵
· - user↵
- id: 3↵
· metadata:↵
··· created: '2024-01-05'
... (truncated)
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC↵
····· country: USA↵
··· name: Grace↵
· tags:↵
· - admin↵
· - verified↵
- id: 2↵
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London↵
····· country: UK↵
··· name: Henry↵
· tags:↵
· - user↵
- id: 3↵
· metadata:↵
··· created: '2024-01-05'
... (truncated)
TOON
chars: 881tokens: 434chars_og/tok: 3.0encoded_chars/tok: 2.0
[5]:↵
· - id: 1↵
··· profile:↵
····· name: Grace↵
····· location:↵
······· city: NYC↵
······· country: USA↵
··· tags[2]: admin,verified↵
··· metadata:↵
····· created: 2024-01-15↵
· - id: 2↵
··· profile:↵
····· name: Henry↵
····· location:↵
······· city: London↵
······· country: UK↵
··· tags[1]: user↵
··· metadata:↵
····· created: 2024-02-20↵
····· updated: 2024-03-10↵
· - id: 3↵
··· profile:↵
····· name: Ivy↵
····· location:↵
······· city: Tokyo
... (truncated)
· - id: 1↵
··· profile:↵
····· name: Grace↵
····· location:↵
······· city: NYC↵
······· country: USA↵
··· tags[2]: admin,verified↵
··· metadata:↵
····· created: 2024-01-15↵
· - id: 2↵
··· profile:↵
····· name: Henry↵
····· location:↵
······· city: London↵
······· country: UK↵
··· tags[1]: user↵
··· metadata:↵
····· created: 2024-02-20↵
····· updated: 2024-03-10↵
· - id: 3↵
··· profile:↵
····· name: Ivy↵
····· location:↵
······· city: Tokyo
... (truncated)
TSON
chars: 453tokens: 207chars_og/tok: 6.4encoded_chars/tok: 2.2
{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 159chars_og/tok: 8.3encoded_chars/tok: 2.6
id; profile{ name; location{ city; country}}; tags[]; metadata{ created; ...}↵
1; { Grace; { NYC; USA}}; [ admin; verified]; { 2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; { 2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; { 2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; { 2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; { 2023-12-01; updated:2024-02-15}
1; { Grace; { NYC; USA}}; [ admin; verified]; { 2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; { 2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; { 2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; { 2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; { 2023-12-01; updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 173chars_og/tok: 7.6encoded_chars/tok: 2.1
id;profile{ name;location{ city;country}};tags[];metadata{ created;...}↵
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
simple_flat.json — Original size: 763 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 763tokens: 334chars_og/tok: 2.3encoded_chars/tok: 2.3
<s>[↵
{↵
··"id":1,↵
··"name":"Alice",↵
··"role":"Engineer",↵
··"department":"Backend"↵
},↵
{↵
··"id":2,↵
··"name":"Bob",↵
··"role":"Designer",↵
··"department":"Frontend"↵
},↵
{↵
··"id":3,↵
··"name":"Carol",↵
··"role":"Manager",↵
··"department":"Product"↵
},↵
{↵
··"id":4,↵
··"name":"David",↵
··"role":"Engineer",↵
··"department":"Infrastructure"↵
},
... (truncated)
{↵
··"id":1,↵
··"name":"Alice",↵
··"role":"Engineer",↵
··"department":"Backend"↵
},↵
{↵
··"id":2,↵
··"name":"Bob",↵
··"role":"Designer",↵
··"department":"Frontend"↵
},↵
{↵
··"id":3,↵
··"name":"Carol",↵
··"role":"Manager",↵
··"department":"Product"↵
},↵
{↵
··"id":4,↵
··"name":"David",↵
··"role":"Engineer",↵
··"department":"Infrastructure"↵
},
... (truncated)
JSON (min)
chars: 522tokens: 165chars_og/tok: 4.6encoded_chars/tok: 3.2
<s>[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 101chars_og/tok: 7.6encoded_chars/tok: 2.3
<s>id,name,role,department
↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
TSV
chars: 234tokens: 101chars_og/tok: 7.6encoded_chars/tok: 2.3
<s>id→name→role→department
↵
1→Alice→Engineer→Backend ↵
2→Bob→Designer→Frontend ↵
3→Carol→Manager→Product ↵
4→David→Engineer→Infrastructure ↵
5→Eva→Analyst→Data ↵
6→Frank→Engineer→Backend ↵
7→Grace→Designer→Mobile ↵
8→Henry→Manager→Engineering ↵
1→Alice→Engineer→Backend ↵
2→Bob→Designer→Frontend ↵
3→Carol→Manager→Product ↵
4→David→Engineer→Infrastructure ↵
5→Eva→Analyst→Data ↵
6→Frank→Engineer→Backend ↵
7→Grace→Designer→Mobile ↵
8→Henry→Manager→Engineering ↵
YAML
chars: 489tokens: 180chars_og/tok: 4.2encoded_chars/tok: 2.7
<s>-department:Backend↵
id:1↵
name:Alice↵
role:Engineer↵
-department:Frontend↵
id:2↵
name:Bob↵
role:Designer↵
-department:Product↵
id:3↵
name:Carol↵
role:Manager↵
-department:Infrastructure↵
id:4↵
name:David↵
role:Engineer↵
-department:Data↵
id:5↵
name:Eva↵
role:Analyst↵
-department:Backend↵
id:6↵
name:Frank↵
role:Engineer↵
-department:Mobile
... (truncated)
id:1↵
name:Alice↵
role:Engineer↵
-department:Frontend↵
id:2↵
name:Bob↵
role:Designer↵
-department:Product↵
id:3↵
name:Carol↵
role:Manager↵
-department:Infrastructure↵
id:4↵
name:David↵
role:Engineer↵
-department:Data↵
id:5↵
name:Eva↵
role:Analyst↵
-department:Backend↵
id:6↵
name:Frank↵
role:Engineer↵
-department:Mobile
... (truncated)
TOON
chars: 246tokens: 103chars_og/tok: 7.4encoded_chars/tok: 2.4
<s>[8]{id,name,role,department}:↵
·1,Alice,Engineer,Backend↵
·2,Bob,Designer,Frontend↵
·3,Carol,Manager,Product↵
·4,David,Engineer,Infrastructure↵
·5,Eva,Analyst,Data↵
·6,Frank,Engineer,Backend↵
·7,Grace,Designer,Mobile↵
·8,Henry,Manager,Engineering
·1,Alice,Engineer,Backend↵
·2,Bob,Designer,Frontend↵
·3,Carol,Manager,Product↵
·4,David,Engineer,Infrastructure↵
·5,Eva,Analyst,Data↵
·6,Frank,Engineer,Backend↵
·7,Grace,Designer,Mobile↵
·8,Henry,Manager,Engineering
TSON
chars: 229tokens: 95chars_og/tok: 8.0encoded_chars/tok: 2.4
<s>{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 83chars_og/tok: 9.2encoded_chars/tok: 3.0
<s>id;name;role;department↵
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
minemizer (compact)
chars: 224tokens: 91chars_og/tok: 8.4encoded_chars/tok: 2.5
<s>id;name;role;department↵
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 741tokens: 322chars_og/tok: 2.3encoded_chars/tok: 2.3
<s>[↵
{↵
··"id":1,↵
··"user":{↵
····"name":"Alice",↵
····"email":"alice@example.com"↵
··},↵
··"status":"active"↵
},↵
{↵
··"id":2,↵
··"user":{↵
····"name":"Bob",↵
····"email":"bob@example.com"↵
··},↵
··"status":"inactive"↵
},↵
{↵
··"id":3,↵
··"user":{↵
····"name":"Carol",↵
····"email":"carol@example.com"↵
··},↵
··"status":"active"↵
},
... (truncated)
{↵
··"id":1,↵
··"user":{↵
····"name":"Alice",↵
····"email":"alice@example.com"↵
··},↵
··"status":"active"↵
},↵
{↵
··"id":2,↵
··"user":{↵
····"name":"Bob",↵
····"email":"bob@example.com"↵
··},↵
··"status":"inactive"↵
},↵
{↵
··"id":3,↵
··"user":{↵
····"name":"Carol",↵
····"email":"carol@example.com"↵
··},↵
··"status":"active"↵
},
... (truncated)
JSON (min)
chars: 470tokens: 159chars_og/tok: 4.7encoded_chars/tok: 3.0
<s>[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 182chars_og/tok: 4.1encoded_chars/tok: 2.5
<s>-id:1↵
status:active↵
user:↵
··email:alice@example.com↵
··name:Alice↵
-id:2↵
status:inactive↵
user:↵
··email:bob@example.com↵
··name:Bob↵
-id:3↵
status:active↵
user:↵
··email:carol@example.com↵
··name:Carol↵
-id:4↵
status:pending↵
user:↵
··email:david@example.com↵
··name:David↵
-id:5↵
status:active↵
user:↵
··email:eva@example.com↵
··name:Eva
... (truncated)
status:active↵
user:↵
··email:alice@example.com↵
··name:Alice↵
-id:2↵
status:inactive↵
user:↵
··email:bob@example.com↵
··name:Bob↵
-id:3↵
status:active↵
user:↵
··email:carol@example.com↵
··name:Carol↵
-id:4↵
status:pending↵
user:↵
··email:david@example.com↵
··name:David↵
-id:5↵
status:active↵
user:↵
··email:eva@example.com↵
··name:Eva
... (truncated)
TOON
chars: 527tokens: 191chars_og/tok: 3.9encoded_chars/tok: 2.8
<s>[6]:↵
-id:1↵
··user:↵
····name:Alice↵
····email:alice@example.com↵
··status:active↵
-id:2↵
··user:↵
····name:Bob↵
····email:bob@example.com↵
··status:inactive↵
-id:3↵
··user:↵
····name:Carol↵
····email:carol@example.com↵
··status:active↵
-id:4↵
··user:↵
····name:David↵
····email:david@example.com↵
··status:pending↵
-id:5↵
··user:↵
····name:Eva↵
····email:eva@example.com
... (truncated)
-id:1↵
··user:↵
····name:Alice↵
····email:alice@example.com↵
··status:active↵
-id:2↵
··user:↵
····name:Bob↵
····email:bob@example.com↵
··status:inactive↵
-id:3↵
··user:↵
····name:Carol↵
····email:carol@example.com↵
··status:active↵
-id:4↵
··user:↵
····name:David↵
····email:david@example.com↵
··status:pending↵
-id:5↵
··user:↵
····name:Eva↵
····email:eva@example.com
... (truncated)
TSON
chars: 249tokens: 104chars_og/tok: 7.1encoded_chars/tok: 2.4
<s>{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 95chars_og/tok: 7.8encoded_chars/tok: 2.7
<s>id;user{name;email};status↵
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
minemizer (compact)
chars: 232tokens: 100chars_og/tok: 7.4encoded_chars/tok: 2.3
<s>id;user{name;email};status↵
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 610tokens: 280chars_og/tok: 2.2encoded_chars/tok: 2.2
<s>[↵
{↵
··"id":1,↵
··"name":"Alice",↵
··"skills":[↵
····"python",↵
····"go",↵
····"rust"↵
··]↵
},↵
{↵
··"id":2,↵
··"name":"Bob",↵
··"skills":[↵
····"javascript",↵
····"typescript"↵
··]↵
},↵
{↵
··"id":3,↵
··"name":"Carol",↵
··"skills":[↵
····"java",↵
····"kotlin",↵
····"scala",
... (truncated)
{↵
··"id":1,↵
··"name":"Alice",↵
··"skills":[↵
····"python",↵
····"go",↵
····"rust"↵
··]↵
},↵
{↵
··"id":2,↵
··"name":"Bob",↵
··"skills":[↵
····"javascript",↵
····"typescript"↵
··]↵
},↵
{↵
··"id":3,↵
··"name":"Carol",↵
··"skills":[↵
····"java",↵
····"kotlin",↵
····"scala",
... (truncated)
JSON (min)
chars: 330tokens: 125chars_og/tok: 4.9encoded_chars/tok: 2.6
<s>[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 157chars_og/tok: 3.9encoded_chars/tok: 2.2
<s>-id:1↵
name:Alice↵
skills:↵
-python↵
-go↵
-rust↵
-id:2↵
name:Bob↵
skills:↵
-javascript↵
-typescript↵
-id:3↵
name:Carol↵
skills:↵
-java↵
-kotlin↵
-scala↵
-groovy↵
-id:4↵
name:David↵
skills:↵
-c↵
-cpp↵
-id:5↵
name:Eva
... (truncated)
name:Alice↵
skills:↵
-python↵
-go↵
-rust↵
-id:2↵
name:Bob↵
skills:↵
-javascript↵
-typescript↵
-id:3↵
name:Carol↵
skills:↵
-java↵
-kotlin↵
-scala↵
-groovy↵
-id:4↵
name:David↵
skills:↵
-c↵
-cpp↵
-id:5↵
name:Eva
... (truncated)
TOON
chars: 339tokens: 141chars_og/tok: 4.3encoded_chars/tok: 2.4
<s>[6]:↵
-id:1↵
··name:Alice↵
··skills[3]:python,go,rust↵
-id:2↵
··name:Bob↵
··skills[2]:javascript,typescript↵
-id:3↵
··name:Carol↵
··skills[4]:java,kotlin,scala,groovy↵
-id:4↵
··name:David↵
··skills[2]:c,cpp↵
-id:5↵
··name:Eva↵
··skills[3]:ruby,elixir,erlang↵
-id:6↵
··name:Frank↵
··skills[1]:swift
-id:1↵
··name:Alice↵
··skills[3]:python,go,rust↵
-id:2↵
··name:Bob↵
··skills[2]:javascript,typescript↵
-id:3↵
··name:Carol↵
··skills[4]:java,kotlin,scala,groovy↵
-id:4↵
··name:David↵
··skills[2]:c,cpp↵
-id:5↵
··name:Eva↵
··skills[3]:ruby,elixir,erlang↵
-id:6↵
··name:Frank↵
··skills[1]:swift
TSON
chars: 168tokens: 79chars_og/tok: 7.7encoded_chars/tok: 2.1
<s>{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 79chars_og/tok: 7.7encoded_chars/tok: 2.5
<s>id;name;skills[]↵
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
minemizer (compact)
chars: 165tokens: 83chars_og/tok: 7.3encoded_chars/tok: 2.0
<s>id;name;skills[]↵
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 589tokens: 278chars_og/tok: 2.1encoded_chars/tok: 2.1
<s>[↵
{↵
··"id":1,↵
··"name":"Carol",↵
··"role":"Manager"↵
},↵
{↵
··"id":2,↵
··"name":"Dave",↵
··"remote":true↵
},↵
{↵
··"id":3,↵
··"name":"Eve",↵
··"role":"Designer",↵
··"team":"UX"↵
},↵
{↵
··"id":4,↵
··"name":"Frank",↵
··"department":"Engineering"↵
},↵
{↵
··"id":5,↵
··"name":"Grace",
... (truncated)
{↵
··"id":1,↵
··"name":"Carol",↵
··"role":"Manager"↵
},↵
{↵
··"id":2,↵
··"name":"Dave",↵
··"remote":true↵
},↵
{↵
··"id":3,↵
··"name":"Eve",↵
··"role":"Designer",↵
··"team":"UX"↵
},↵
{↵
··"id":4,↵
··"name":"Frank",↵
··"department":"Engineering"↵
},↵
{↵
··"id":5,↵
··"name":"Grace",
... (truncated)
JSON (min)
chars: 378tokens: 133chars_og/tok: 4.4encoded_chars/tok: 2.8
<s>[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 150chars_og/tok: 3.9encoded_chars/tok: 2.4
<s>-id:1↵
name:Carol↵
role:Manager↵
-id:2↵
name:Dave↵
remote:true↵
-id:3↵
name:Eve↵
role:Designer↵
team:UX↵
-department:Engineering↵
id:4↵
name:Frank↵
-id:5↵
name:Grace↵
remote:true↵
role:Engineer↵
team:Platform↵
-id:6↵
name:Henry↵
role:Analyst↵
-id:7↵
name:Ivy↵
-department:Sales↵
id:8
... (truncated)
name:Carol↵
role:Manager↵
-id:2↵
name:Dave↵
remote:true↵
-id:3↵
name:Eve↵
role:Designer↵
team:UX↵
-department:Engineering↵
id:4↵
name:Frank↵
-id:5↵
name:Grace↵
remote:true↵
role:Engineer↵
team:Platform↵
-id:6↵
name:Henry↵
role:Analyst↵
-id:7↵
name:Ivy↵
-department:Sales↵
id:8
... (truncated)
TOON
chars: 414tokens: 161chars_og/tok: 3.7encoded_chars/tok: 2.6
<s>[8]:↵
-id:1↵
··name:Carol↵
··role:Manager↵
-id:2↵
··name:Dave↵
··remote:true↵
-id:3↵
··name:Eve↵
··role:Designer↵
··team:UX↵
-id:4↵
··name:Frank↵
··department:Engineering↵
-id:5↵
··name:Grace↵
··role:Engineer↵
··remote:true↵
··team:Platform↵
-id:6↵
··name:Henry↵
··role:Analyst↵
-id:7↵
··name:Ivy↵
-id:8
... (truncated)
-id:1↵
··name:Carol↵
··role:Manager↵
-id:2↵
··name:Dave↵
··remote:true↵
-id:3↵
··name:Eve↵
··role:Designer↵
··team:UX↵
-id:4↵
··name:Frank↵
··department:Engineering↵
-id:5↵
··name:Grace↵
··role:Engineer↵
··remote:true↵
··team:Platform↵
-id:6↵
··name:Henry↵
··role:Analyst↵
-id:7↵
··name:Ivy↵
-id:8
... (truncated)
TSON
chars: 300tokens: 133chars_og/tok: 4.4encoded_chars/tok: 2.3
<s>[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 87chars_og/tok: 6.8encoded_chars/tok: 2.7
<s>id;name;role↵
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
minemizer (compact)
chars: 207tokens: 90chars_og/tok: 6.5encoded_chars/tok: 2.3
<s>id;name;role↵
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 1,611,780tokens: 749,813chars_og/tok: 2.1encoded_chars/tok: 2.1
<s>[↵
{↵
··"id":"_",↵
··"symbol":"gib",↵
··"name":"\u0f3c\u3064\u25d5_\u25d5\u0f3d\u3064"↵
},↵
{↵
··"id":"000-capital",↵
··"symbol":"000",↵
··"name":"000Capital"↵
},↵
{↵
··"id":"01111010011110000110001001110100-token",↵
··"symbol":"01111010011110000110001001110100",↵
··"name":"01111010011110000110001001110100"↵
},↵
{↵
··"id":"01-token",↵
··"symbol":"01",↵
··"name":"01"↵
},↵
{↵
··"id":"0chain",↵
··"symbol":"zcn",↵
··"name":"Zus"
... (truncated)
{↵
··"id":"_",↵
··"symbol":"gib",↵
··"name":"\u0f3c\u3064\u25d5_\u25d5\u0f3d\u3064"↵
},↵
{↵
··"id":"000-capital",↵
··"symbol":"000",↵
··"name":"000Capital"↵
},↵
{↵
··"id":"01111010011110000110001001110100-token",↵
··"symbol":"01111010011110000110001001110100",↵
··"name":"01111010011110000110001001110100"↵
},↵
{↵
··"id":"01-token",↵
··"symbol":"01",↵
··"name":"01"↵
},↵
{↵
··"id":"0chain",↵
··"symbol":"zcn",↵
··"name":"Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 422,810chars_og/tok: 3.8encoded_chars/tok: 2.7
<s>[{"id":"_","symbol":"gib","name":"\u0f3c\u3064\u25d5_\u25d5\u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIXProtocol"},{"id":"0x","symbol":"zrx","name":"0xProtocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai:AISmartContract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0xLeverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonkbyVirtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"AvalancheBridgedZRX(Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsimbyVirtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS(Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000xbyVirtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1CoinCanChangeYourLife"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1communitycanchangeyourlife"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1dogcanchangeyourlife"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1HiveWater"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCHyVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1MoveToken"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1narrativecanchangeyourlife"},{"id":"1-one","symbol":"one","name":"1(one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUSDAO"},{"id":"1-squirrel","symbol":"peanut","name":"OGPeanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004PEPE"},{"id":"2025-token","symbol":"2025","name":"2025TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3yearsandforget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24KGoldPEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2GCarbonCoin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"MeowMeowCoin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACEART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
... (truncated)
CSV
chars: 603,635tokens: 303,115chars_og/tok: 5.3encoded_chars/tok: 2.0
<s>id,symbol,name
↵
_,gib,���つ���_������つ ↵
000-capital,000,000Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIXProtocol ↵
0x,zrx,0xProtocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai:AISmartContract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0xLeverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonkbyVirtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,AvalancheBridgedZRX(Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsimbyVirtuals ↵
0xy,0xy,0xy ↵
-10,loong,龙 ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
_,gib,���つ���_������つ ↵
000-capital,000,000Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIXProtocol ↵
0x,zrx,0xProtocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai:AISmartContract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0xLeverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonkbyVirtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,AvalancheBridgedZRX(Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsimbyVirtuals ↵
0xy,0xy,0xy ↵
-10,loong,龙 ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 303,547chars_og/tok: 5.3encoded_chars/tok: 2.0
<s>id→symbol→name
↵
_→gib→���つ���_������つ ↵
000-capital→000→000Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain→zcn→Zus ↵
0vix-protocol→vix→0VIXProtocol ↵
0x→zrx→0xProtocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai:AISmartContract ↵
0x678-landwolf-1933→wolf→Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen→xgn→0xGen ↵
0x-leverage→oxl→0xLeverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonkbyVirtuals ↵
0x-protocol-avalanche-bridged-zrx-e→zrx.e→AvalancheBridgedZRX(Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals→sage→0xsimbyVirtuals ↵
0xy→0xy→0xy ↵
-10→loong→龙 ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
_→gib→���つ���_������つ ↵
000-capital→000→000Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain→zcn→Zus ↵
0vix-protocol→vix→0VIXProtocol ↵
0x→zrx→0xProtocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai:AISmartContract ↵
0x678-landwolf-1933→wolf→Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen→xgn→0xGen ↵
0x-leverage→oxl→0xLeverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonkbyVirtuals ↵
0x-protocol-avalanche-bridged-zrx-e→zrx.e→AvalancheBridgedZRX(Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals→sage→0xsimbyVirtuals ↵
0xy→0xy→0xy ↵
-10→loong→龙 ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 454,247chars_og/tok: 3.5encoded_chars/tok: 2.3
<s>-id:_↵
name:���つ���_������つ↵
symbol:gib↵
-id:000-capital↵
name:000Capital↵
symbol:'000'↵
-id:01111010011110000110001001110100-token↵
name:'01111010011110000110001001110100'↵
symbol:'01111010011110000110001001110100'↵
-id:01-token↵
name:'01'↵
symbol:'01'↵
-id:0chain↵
name:Zus↵
symbol:zcn↵
-id:0vix-protocol↵
name:0VIXProtocol↵
symbol:vix↵
-id:0x↵
name:0xProtocol↵
symbol:zrx↵
-id:0x0-ai-ai-smart-contract↵
name:'0x0.ai:AISmartContract'↵
symbol:'0x0'↵
-id:0x678-landwolf-1933
... (truncated)
name:���つ���_������つ↵
symbol:gib↵
-id:000-capital↵
name:000Capital↵
symbol:'000'↵
-id:01111010011110000110001001110100-token↵
name:'01111010011110000110001001110100'↵
symbol:'01111010011110000110001001110100'↵
-id:01-token↵
name:'01'↵
symbol:'01'↵
-id:0chain↵
name:Zus↵
symbol:zcn↵
-id:0vix-protocol↵
name:0VIXProtocol↵
symbol:vix↵
-id:0x↵
name:0xProtocol↵
symbol:zrx↵
-id:0x0-ai-ai-smart-contract↵
name:'0x0.ai:AISmartContract'↵
symbol:'0x0'↵
-id:0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 301,477chars_og/tok: 5.3encoded_chars/tok: 2.1
<s>[19332]{id,symbol,name}:↵
_,gib,���つ���_������つ↵
·000-capital,"000",000Capital↵
·01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
·01-token,"01","01"↵
·0chain,zcn,Zus↵
·0vix-protocol,vix,0VIXProtocol↵
·0x,zrx,0xProtocol↵
·0x0-ai-ai-smart-contract,0x0,"0x0.ai:AISmartContract"↵
·0x678-landwolf-1933,wolf,Landwolf↵
·0xgasless-2,0xgas,0xGasless↵
·0xgen,xgn,0xGen↵
·0x-leverage,oxl,0xLeverage↵
·0xlsd,0xlsd,0xLSD↵
·0xmonk,monk,0xMonkbyVirtuals↵
·0x-protocol-avalanche-bridged-zrx-e,zrx.e,AvalancheBridgedZRX(Avalanche)↵
·0xshadow,0xs,0xShadow↵
·0xsim-by-virtuals,sage,0xsimbyVirtuals↵
·0xy,0xy,0xy↵
"-10",loong,龙↵
·1000bonk,1000bonk,1000BONK↵
·1000btt,1000btt,1000BTT↵
·1000cat,1000cat,1000CAT↵
·1000chems,1000cheems,1000CHEMS↵
·1000mog,1000mog,1000MOG
... (truncated)
_,gib,���つ���_������つ↵
·000-capital,"000",000Capital↵
·01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
·01-token,"01","01"↵
·0chain,zcn,Zus↵
·0vix-protocol,vix,0VIXProtocol↵
·0x,zrx,0xProtocol↵
·0x0-ai-ai-smart-contract,0x0,"0x0.ai:AISmartContract"↵
·0x678-landwolf-1933,wolf,Landwolf↵
·0xgasless-2,0xgas,0xGasless↵
·0xgen,xgn,0xGen↵
·0x-leverage,oxl,0xLeverage↵
·0xlsd,0xlsd,0xLSD↵
·0xmonk,monk,0xMonkbyVirtuals↵
·0x-protocol-avalanche-bridged-zrx-e,zrx.e,AvalancheBridgedZRX(Avalanche)↵
·0xshadow,0xs,0xShadow↵
·0xsim-by-virtuals,sage,0xsimbyVirtuals↵
·0xy,0xy,0xy↵
"-10",loong,龙↵
·1000bonk,1000bonk,1000BONK↵
·1000btt,1000btt,1000BTT↵
·1000cat,1000cat,1000CAT↵
·1000chems,1000cheems,1000CHEMS↵
·1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 293,257chars_og/tok: 5.5encoded_chars/tok: 2.1
<s>{@id,symbol,name#19332|_,gib,"���つ���_������つ"|000-capital,"000","000Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIXProtocol"|0x,zrx,"0xProtocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai:AISmartContract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0xLeverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonkbyVirtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"AvalancheBridgedZRX(Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsimbyVirtuals"|0xy,0xy,0xy|"-10",loong,龙|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS(Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000xbyVirtuals"|100-token,100¥,100¥|100xdarren,100x,100xDarren|10-figs,figs,"10figs"|"-11",���长���,���长���|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1CoinCanChangeYourLife"|1-community-can-change-your-life,community,"1communitycanchangeyourlife"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1dogcanchangeyourlife"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1HiveWater"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCHyVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1MoveToken"|1-narrative-can-change-your-life,narrative,"1narrativecanchangeyourlife"|1-one,one,"1(one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUSDAO"|1-squirrel,peanut,"OGPeanut"|1-token,"1","1"|2004-pepe,bog,"2004PEPE"|2025-token,"2025","2025TOKEN"|2077-code,"2077","2077CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23Turtles"|2-3-years-and-forget,"23","2-3yearsandforget"|24k-gold-pepe,goldpepe,"24KGoldPEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2GCarbonCoin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"MeowMeowCoin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACEART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404—GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47thPOTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4NextUnicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TBCoin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-WayMirrorMoney"|"-5",��������,������������������������|500m-piece-of-paper,paper,"$500Mpieceofpaper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5thScape"|"-6"," "," "|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",∅,Voidify|717ai-by-virtuals,wire,"717aibyVirtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",����,����|8004-dog,dog8004,"8004Dog"|8008-token,"8008","8008"|888coin,���������,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8Ball"|8-bit-coin,coin,"8-BitCoin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",∑,∑|9-5,9-5,9to5|99-bitcoins,99btc,"99Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16zAIDog"|a51-finance,a51,"A51Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaacat"|aaai_agent-by-virtuals,aaai,"AAAI_agentbyVirtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADexFinance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"AarkDigital"|aarna-afi-802v2,"afi802v2","aarnaafi802v2"|aarna-atv111,atv111,"aarnaatv111"|aarna-atv111-arbitrum,atv111,"aarnaatv111(Arbitrum)"|aarna-atv111-sonic,atv111,"aarnaatv111(Sonic)"|aarna-atv-808,atv808,"aarnaatv808"|aarna-atv-usdc,atvusdc,"aarnaatvUSDC(Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarnaatvUSDC(Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"AaveAAVE"|aave-amm-bptbalweth,aammbptbalweth,"AaveAMMBptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"AaveAMMBptWBTCWETH"|aave-amm-dai,aammdai,"AaveAMMDAI"|aave-amm-uniaaveweth,aammuniaaveweth,"AaveAMMUniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"AaveAMMUniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"AaveAMMUniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"AaveAMMUniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"AaveAMMUniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"AaveAMMUniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"AaveAMM
... (truncated)
... (truncated)
minemizer
chars: 622,925tokens: 282,585chars_og/tok: 5.7encoded_chars/tok: 2.2
<s>id;symbol;name↵
_;gib;���つ���_������つ↵
000-capital;000;000Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIXProtocol↵
0x;zrx;0xProtocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai:AISmartContract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0xLeverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonkbyVirtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;AvalancheBridgedZRX(Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsimbyVirtuals↵
0xy;0xy;0xy↵
-10;loong;龙↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
_;gib;���つ���_������つ↵
000-capital;000;000Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIXProtocol↵
0x;zrx;0xProtocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai:AISmartContract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0xLeverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonkbyVirtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;AvalancheBridgedZRX(Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsimbyVirtuals↵
0xy;0xy;0xy↵
-10;loong;龙↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 285,149chars_og/tok: 5.7encoded_chars/tok: 2.0
<s>id;symbol;name↵
_;gib;���つ���_������つ↵
000-capital;000;000Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIXProtocol↵
0x;zrx;0xProtocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai:AISmartContract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0xLeverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonkbyVirtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;AvalancheBridgedZRX(Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsimbyVirtuals↵
0xy;0xy;0xy↵
-10;loong;龙↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
_;gib;���つ���_������つ↵
000-capital;000;000Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIXProtocol↵
0x;zrx;0xProtocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai:AISmartContract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0xLeverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonkbyVirtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;AvalancheBridgedZRX(Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsimbyVirtuals↵
0xy;0xy;0xy↵
-10;loong;龙↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: llama (NousResearch/Llama-2-7b-hf)
JSON (pretty)
chars: 1,320tokens: 560chars_og/tok: 2.4encoded_chars/tok: 2.4
<s>[↵
{↵
··"id":1,↵
··"profile":{↵
····"name":"Grace",↵
····"location":{↵
······"city":"NYC",↵
······"country":"USA"↵
····}↵
··},↵
··"tags":[↵
····"admin",↵
····"verified"↵
··],↵
··"metadata":{↵
····"created":"2024-01-15"↵
··}↵
},↵
{↵
··"id":2,↵
··"profile":{↵
····"name":"Henry",↵
····"location":{↵
······"city":"London",↵
······"country":"UK"
... (truncated)
{↵
··"id":1,↵
··"profile":{↵
····"name":"Grace",↵
····"location":{↵
······"city":"NYC",↵
······"country":"USA"↵
····}↵
··},↵
··"tags":[↵
····"admin",↵
····"verified"↵
··],↵
··"metadata":{↵
····"created":"2024-01-15"↵
··}↵
},↵
{↵
··"id":2,↵
··"profile":{↵
····"name":"Henry",↵
····"location":{↵
······"city":"London",↵
······"country":"UK"
... (truncated)
JSON (min)
chars: 760tokens: 284chars_og/tok: 4.6encoded_chars/tok: 2.7
<s>[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 338chars_og/tok: 3.9encoded_chars/tok: 2.4
<s>-id:1↵
metadata:↵
··created:'2024-01-15'↵
profile:↵
··location:↵
····city:NYC↵
····country:USA↵
··name:Grace↵
tags:↵
-admin↵
-verified↵
-id:2↵
metadata:↵
··created:'2024-02-20'↵
··updated:'2024-03-10'↵
profile:↵
··location:↵
····city:London↵
····country:UK↵
··name:Henry↵
tags:↵
-user↵
-id:3↵
metadata:↵
··created:'2024-01-05'
... (truncated)
metadata:↵
··created:'2024-01-15'↵
profile:↵
··location:↵
····city:NYC↵
····country:USA↵
··name:Grace↵
tags:↵
-admin↵
-verified↵
-id:2↵
metadata:↵
··created:'2024-02-20'↵
··updated:'2024-03-10'↵
profile:↵
··location:↵
····city:London↵
····country:UK↵
··name:Henry↵
tags:↵
-user↵
-id:3↵
metadata:↵
··created:'2024-01-05'
... (truncated)
TOON
chars: 881tokens: 329chars_og/tok: 4.0encoded_chars/tok: 2.7
<s>[5]:↵
-id:1↵
··profile:↵
····name:Grace↵
····location:↵
······city:NYC↵
······country:USA↵
··tags[2]:admin,verified↵
··metadata:↵
····created:2024-01-15↵
-id:2↵
··profile:↵
····name:Henry↵
····location:↵
······city:London↵
······country:UK↵
··tags[1]:user↵
··metadata:↵
····created:2024-02-20↵
····updated:2024-03-10↵
-id:3↵
··profile:↵
····name:Ivy↵
····location:↵
······city:Tokyo
... (truncated)
-id:1↵
··profile:↵
····name:Grace↵
····location:↵
······city:NYC↵
······country:USA↵
··tags[2]:admin,verified↵
··metadata:↵
····created:2024-01-15↵
-id:2↵
··profile:↵
····name:Henry↵
····location:↵
······city:London↵
······country:UK↵
··tags[1]:user↵
··metadata:↵
····created:2024-02-20↵
····updated:2024-03-10↵
-id:3↵
··profile:↵
····name:Ivy↵
····location:↵
······city:Tokyo
... (truncated)
TSON
chars: 453tokens: 237chars_og/tok: 5.6encoded_chars/tok: 1.9
<s>{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 201chars_og/tok: 6.6encoded_chars/tok: 2.1
<s>id;profile{name;location{city;country}};tags[];metadata{created;...}↵
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 214chars_og/tok: 6.2encoded_chars/tok: 1.7
<s>id;profile{name;location{city;country}};tags[];metadata{created;...}↵
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
simple_flat.json — Original size: 763 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 763tokens: 264chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
JSON (min)
chars: 522tokens: 137chars_og/tok: 5.6encoded_chars/tok: 3.8
[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id,name,role,department
↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
TSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id name→role→department
↵
1 Alice→Engineer→Backend ↵
2 Bob→Designer Frontend ↵
3 Carol→Manager→Product ↵
4 David→Engineer→Infrastructure ↵
5 Eva→Analyst Data ↵
6 Frank→Engineer→Backend ↵
7→Grace→Designer Mobile ↵
8 Henry→Manager→Engineering ↵
1 Alice→Engineer→Backend ↵
2 Bob→Designer Frontend ↵
3 Carol→Manager→Product ↵
4 David→Engineer→Infrastructure ↵
5 Eva→Analyst Data ↵
6 Frank→Engineer→Backend ↵
7→Grace→Designer Mobile ↵
8 Henry→Manager→Engineering ↵
YAML
chars: 489tokens: 169chars_og/tok: 4.5encoded_chars/tok: 2.9
- department: Backend↵
· id:·1↵
· name: Alice↵
· role: Engineer↵
- department: Frontend↵
· id:·2↵
· name: Bob↵
· role: Designer↵
- department: Product↵
· id:·3↵
· name: Carol↵
· role: Manager↵
- department: Infrastructure↵
· id:·4↵
· name: David↵
· role: Engineer↵
- department: Data↵
· id:·5↵
· name: Eva↵
· role: Analyst↵
- department: Backend↵
· id:·6↵
· name: Frank↵
· role: Engineer↵
- department: Mobile
... (truncated)
· id:·1↵
· name: Alice↵
· role: Engineer↵
- department: Frontend↵
· id:·2↵
· name: Bob↵
· role: Designer↵
- department: Product↵
· id:·3↵
· name: Carol↵
· role: Manager↵
- department: Infrastructure↵
· id:·4↵
· name: David↵
· role: Engineer↵
- department: Data↵
· id:·5↵
· name: Eva↵
· role: Analyst↵
- department: Backend↵
· id:·6↵
· name: Frank↵
· role: Engineer↵
- department: Mobile
... (truncated)
TOON
chars: 246tokens: 96chars_og/tok: 7.9encoded_chars/tok: 2.6
[8]{id,name,role,department}:↵
··1,Alice,Engineer,Backend↵
··2,Bob,Designer,Frontend↵
··3,Carol,Manager,Product↵
··4,David,Engineer,Infrastructure↵
··5,Eva,Analyst,Data↵
··6,Frank,Engineer,Backend↵
··7,Grace,Designer,Mobile↵
··8,Henry,Manager,Engineering
··1,Alice,Engineer,Backend↵
··2,Bob,Designer,Frontend↵
··3,Carol,Manager,Product↵
··4,David,Engineer,Infrastructure↵
··5,Eva,Analyst,Data↵
··6,Frank,Engineer,Backend↵
··7,Grace,Designer,Mobile↵
··8,Henry,Manager,Engineering
TSON
chars: 229tokens: 80chars_og/tok: 9.5encoded_chars/tok: 2.9
{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 72chars_og/tok: 10.6encoded_chars/tok: 3.5
id; name; role; department↵
1; Alice; Engineer; Backend↵
2; Bob; Designer; Frontend↵
3; Carol; Manager; Product↵
4; David; Engineer; Infrastructure↵
5; Eva; Analyst; Data↵
6; Frank; Engineer; Backend↵
7; Grace; Designer; Mobile↵
8; Henry; Manager; Engineering
1; Alice; Engineer; Backend↵
2; Bob; Designer; Frontend↵
3; Carol; Manager; Product↵
4; David; Engineer; Infrastructure↵
5; Eva; Analyst; Data↵
6; Frank; Engineer; Backend↵
7; Grace; Designer; Mobile↵
8; Henry; Manager; Engineering
minemizer (compact)
chars: 224tokens: 77chars_og/tok: 9.9encoded_chars/tok: 2.9
id;name;role;department↵
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 741tokens: 252chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id":·2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id":·3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
· {↵
··· "id":·1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id":·2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id":·3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
JSON (min)
chars: 470tokens: 127chars_og/tok: 5.8encoded_chars/tok: 3.7
[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 158chars_og/tok: 4.7encoded_chars/tok: 2.9
- id:·1↵
· status: active↵
· user:↵
··· email: alice@example.com↵
··· name: Alice↵
- id:·2↵
· status: inactive↵
· user:↵
··· email: bob@example.com↵
··· name: Bob↵
- id:·3↵
· status: active↵
· user:↵
··· email: carol@example.com↵
··· name: Carol↵
- id:·4↵
· status: pending↵
· user:↵
··· email: david@example.com↵
··· name: David↵
- id:·5↵
· status: active↵
· user:↵
··· email: eva@example.com↵
··· name: Eva
... (truncated)
· status: active↵
· user:↵
··· email: alice@example.com↵
··· name: Alice↵
- id:·2↵
· status: inactive↵
· user:↵
··· email: bob@example.com↵
··· name: Bob↵
- id:·3↵
· status: active↵
· user:↵
··· email: carol@example.com↵
··· name: Carol↵
- id:·4↵
· status: pending↵
· user:↵
··· email: david@example.com↵
··· name: David↵
- id:·5↵
· status: active↵
· user:↵
··· email: eva@example.com↵
··· name: Eva
... (truncated)
TOON
chars: 527tokens: 166chars_og/tok: 4.5encoded_chars/tok: 3.2
[6]:↵
· - id:·1↵
··· user:↵
····· name: Alice↵
····· email: alice@example.com↵
··· status: active↵
· - id:·2↵
··· user:↵
····· name: Bob↵
····· email: bob@example.com↵
··· status: inactive↵
· - id:·3↵
··· user:↵
····· name: Carol↵
····· email: carol@example.com↵
··· status: active↵
· - id:·4↵
··· user:↵
····· name: David↵
····· email: david@example.com↵
··· status: pending↵
· - id:·5↵
··· user:↵
····· name: Eva↵
····· email: eva@example.com
... (truncated)
· - id:·1↵
··· user:↵
····· name: Alice↵
····· email: alice@example.com↵
··· status: active↵
· - id:·2↵
··· user:↵
····· name: Bob↵
····· email: bob@example.com↵
··· status: inactive↵
· - id:·3↵
··· user:↵
····· name: Carol↵
····· email: carol@example.com↵
··· status: active↵
· - id:·4↵
··· user:↵
····· name: David↵
····· email: david@example.com↵
··· status: pending↵
· - id:·5↵
··· user:↵
····· name: Eva↵
····· email: eva@example.com
... (truncated)
TSON
chars: 249tokens: 75chars_og/tok: 9.9encoded_chars/tok: 3.3
{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 77chars_og/tok: 9.6encoded_chars/tok: 3.4
id; user{ name; email}; status↵
1; { Alice; alice@example.com}; active↵
2; { Bob; bob@example.com}; inactive↵
3; { Carol; carol@example.com}; active↵
4; { David; david@example.com}; pending↵
5; { Eva; eva@example.com}; active↵
6; { Frank; frank@example.com}; active
1; { Alice; alice@example.com}; active↵
2; { Bob; bob@example.com}; inactive↵
3; { Carol; carol@example.com}; active↵
4; { David; david@example.com}; pending↵
5; { Eva; eva@example.com}; active↵
6; { Frank; frank@example.com}; active
minemizer (compact)
chars: 232tokens: 78chars_og/tok: 9.5encoded_chars/tok: 3.0
id;user{ name;email};status↵
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 610tokens: 217chars_og/tok: 2.8encoded_chars/tok: 2.8
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
JSON (min)
chars: 330tokens: 103chars_og/tok: 5.9encoded_chars/tok: 3.2
[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 149chars_og/tok: 4.1encoded_chars/tok: 2.3
- id:·1↵
· name: Alice↵
· skills:↵
· - python↵
· - go↵
· - rust↵
- id:·2↵
· name: Bob↵
· skills:↵
· - javascript↵
· - typescript↵
- id:·3↵
· name: Carol↵
· skills:↵
· - java↵
· - kotlin↵
· - scala↵
· - groovy↵
- id:·4↵
· name: David↵
· skills:↵
· - c↵
· - cpp↵
- id:·5↵
· name: Eva
... (truncated)
· name: Alice↵
· skills:↵
· - python↵
· - go↵
· - rust↵
- id:·2↵
· name: Bob↵
· skills:↵
· - javascript↵
· - typescript↵
- id:·3↵
· name: Carol↵
· skills:↵
· - java↵
· - kotlin↵
· - scala↵
· - groovy↵
- id:·4↵
· name: David↵
· skills:↵
· - c↵
· - cpp↵
- id:·5↵
· name: Eva
... (truncated)
TOON
chars: 339tokens: 137chars_og/tok: 4.5encoded_chars/tok: 2.5
[6]:↵
· - id:·1↵
··· name: Alice↵
··· skills[3]: python,go,rust↵
· - id:·2↵
··· name: Bob↵
··· skills[2]: javascript,typescript↵
· - id:·3↵
··· name: Carol↵
··· skills[4]: java,kotlin,scala,groovy↵
· - id:·4↵
··· name: David↵
··· skills[2]: c,cpp↵
· - id:·5↵
··· name: Eva↵
··· skills[3]: ruby,elixir,erlang↵
· - id:·6↵
··· name: Frank↵
··· skills[1]: swift
· - id:·1↵
··· name: Alice↵
··· skills[3]: python,go,rust↵
· - id:·2↵
··· name: Bob↵
··· skills[2]: javascript,typescript↵
· - id:·3↵
··· name: Carol↵
··· skills[4]: java,kotlin,scala,groovy↵
· - id:·4↵
··· name: David↵
··· skills[2]: c,cpp↵
· - id:·5↵
··· name: Eva↵
··· skills[3]: ruby,elixir,erlang↵
· - id:·6↵
··· name: Frank↵
··· skills[1]: swift
TSON
chars: 168tokens: 65chars_og/tok: 9.4encoded_chars/tok: 2.6
{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 71chars_og/tok: 8.6encoded_chars/tok: 2.7
id; name; skills[]↵
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
minemizer (compact)
chars: 165tokens: 70chars_og/tok: 8.7encoded_chars/tok: 2.4
id;name;skills[]↵
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 589tokens: 224chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id":·1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Dave",↵
··· "remote": true↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id":·5,↵
··· "name": "Grace",
... (truncated)
· {↵
··· "id":·1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Dave",↵
··· "remote": true↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id":·5,↵
··· "name": "Grace",
... (truncated)
JSON (min)
chars: 378tokens: 114chars_og/tok: 5.2encoded_chars/tok: 3.3
[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 143chars_og/tok: 4.1encoded_chars/tok: 2.5
- id:·1↵
· name: Carol↵
· role: Manager↵
- id:·2↵
· name: Dave↵
· remote: true↵
- id:·3↵
· name: Eve↵
· role: Designer↵
· team: UX↵
- department: Engineering↵
· id:·4↵
· name: Frank↵
- id:·5↵
· name: Grace↵
· remote: true↵
· role: Engineer↵
· team: Platform↵
- id:·6↵
· name: Henry↵
· role: Analyst↵
- id:·7↵
· name: Ivy↵
- department: Sales↵
· id:·8
... (truncated)
· name: Carol↵
· role: Manager↵
- id:·2↵
· name: Dave↵
· remote: true↵
- id:·3↵
· name: Eve↵
· role: Designer↵
· team: UX↵
- department: Engineering↵
· id:·4↵
· name: Frank↵
- id:·5↵
· name: Grace↵
· remote: true↵
· role: Engineer↵
· team: Platform↵
- id:·6↵
· name: Henry↵
· role: Analyst↵
- id:·7↵
· name: Ivy↵
- department: Sales↵
· id:·8
... (truncated)
TOON
chars: 414tokens: 153chars_og/tok: 3.8encoded_chars/tok: 2.7
[8]:↵
· - id:·1↵
··· name: Carol↵
··· role: Manager↵
· - id:·2↵
··· name: Dave↵
··· remote: true↵
· - id:·3↵
··· name: Eve↵
··· role: Designer↵
··· team: UX↵
· - id:·4↵
··· name: Frank↵
··· department: Engineering↵
· - id:·5↵
··· name: Grace↵
··· role: Engineer↵
··· remote: true↵
··· team: Platform↵
· - id:·6↵
··· name: Henry↵
··· role: Analyst↵
· - id:·7↵
··· name: Ivy↵
· - id:·8
... (truncated)
· - id:·1↵
··· name: Carol↵
··· role: Manager↵
· - id:·2↵
··· name: Dave↵
··· remote: true↵
· - id:·3↵
··· name: Eve↵
··· role: Designer↵
··· team: UX↵
· - id:·4↵
··· name: Frank↵
··· department: Engineering↵
· - id:·5↵
··· name: Grace↵
··· role: Engineer↵
··· remote: true↵
··· team: Platform↵
· - id:·6↵
··· name: Henry↵
··· role: Analyst↵
· - id:·7↵
··· name: Ivy↵
· - id:·8
... (truncated)
TSON
chars: 300tokens: 109chars_og/tok: 5.4encoded_chars/tok: 2.8
[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 77chars_og/tok: 7.6encoded_chars/tok: 3.0
id; name; role↵
1; Carol; Manager↵
2; Dave; ; remote:True↵
3; Eve; Designer; team:UX↵
4; Frank; ; department:Engineering↵
5; Grace; Engineer; remote:True; team:Platform↵
6; Henry; Analyst↵
7; Ivy; ↵
8; Jack; ; department:Sales; remote:False
1; Carol; Manager↵
2; Dave; ; remote:True↵
3; Eve; Designer; team:UX↵
4; Frank; ; department:Engineering↵
5; Grace; Engineer; remote:True; team:Platform↵
6; Henry; Analyst↵
7; Ivy; ↵
8; Jack; ; department:Sales; remote:False
minemizer (compact)
chars: 207tokens: 77chars_og/tok: 7.6encoded_chars/tok: 2.7
id;name;role↵
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 1,611,780tokens: 611,575chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 361,897chars_og/tok: 4.5encoded_chars/tok: 3.2
[{"id":"_","symbol":"gib","name":"\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000 Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIX Protocol"},{"id":"0x","symbol":"zrx","name":"0x Protocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai: AI Smart Contract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0x Leverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonk by Virtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"Avalanche Bridged ZRX (Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsim by Virtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS (Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000x by Virtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10 figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1 Coin Can Change Your Life"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1 community can change your life"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1 dog can change your life"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1Hive Water"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCH yVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1Move Token"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1 narrative can change your life"},{"id":"1-one","symbol":"one","name":"1 (one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUS DAO"},{"id":"1-squirrel","symbol":"peanut","name":"OG Peanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004 PEPE"},{"id":"2025-token","symbol":"2025","name":"2025 TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077 CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23 Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3 years and forget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24K Gold PEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2G Carbon Coin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"Meow Meow Coin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACE ART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
... (truncated)
CSV
chars: 603,635tokens: 228,404chars_og/tok: 7.1encoded_chars/tok: 2.6
id,symbol,name
↵
_,gib,༼·つ ��_◕ ���つ ↵
000-capital,000,000 Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIX Protocol ↵
0x,zrx,0x Protocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0x Leverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsim by Virtuals ↵
0xy,0xy,0xy ↵
-10,loong,龙 ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
_,gib,༼·つ ��_◕ ���つ ↵
000-capital,000,000 Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIX Protocol ↵
0x,zrx,0x Protocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0x Leverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsim by Virtuals ↵
0xy,0xy,0xy ↵
-10,loong,龙 ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 224,078chars_og/tok: 7.2encoded_chars/tok: 2.7
id symbol name
↵
_ gib→༼·つ ��_◕ ���つ ↵
000-capital→000→000 Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain zcn Zus ↵
0vix-protocol vix→0VIX Protocol ↵
0x zrx→0x Protocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933 wolf Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen xgn→0xGen ↵
0x-leverage→oxl→0x Leverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e zrx.e Avalanche Bridged ZRX (Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals sage→0xsim by Virtuals ↵
0xy→0xy→0xy ↵
-10→loong→龙 ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
_ gib→༼·つ ��_◕ ���つ ↵
000-capital→000→000 Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain zcn Zus ↵
0vix-protocol vix→0VIX Protocol ↵
0x zrx→0x Protocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933 wolf Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen xgn→0xGen ↵
0x-leverage→oxl→0x Leverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e zrx.e Avalanche Bridged ZRX (Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals sage→0xsim by Virtuals ↵
0xy→0xy→0xy ↵
-10→loong→龙 ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 409,204chars_og/tok: 3.9encoded_chars/tok: 2.6
- id: _↵
· name: ���·つ ��_◕ ���つ↵
· symbol: gib↵
- id:·000-capital↵
· name:·000 Capital↵
· symbol: '000'↵
- id:·01111010011110000110001001110100-token↵
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id:·01-token↵
· name: '01'↵
· symbol: '01'↵
- id:·0chain↵
· name: Zus↵
· symbol: zcn↵
- id:·0vix-protocol↵
· name:·0VIX Protocol↵
· symbol: vix↵
- id:·0x↵
· name:·0x Protocol↵
· symbol: zrx↵
- id:·0x0-ai-ai-smart-contract↵
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id:·0x678-landwolf-1933
... (truncated)
· name: ���·つ ��_◕ ���つ↵
· symbol: gib↵
- id:·000-capital↵
· name:·000 Capital↵
· symbol: '000'↵
- id:·01111010011110000110001001110100-token↵
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id:·01-token↵
· name: '01'↵
· symbol: '01'↵
- id:·0chain↵
· name: Zus↵
· symbol: zcn↵
- id:·0vix-protocol↵
· name:·0VIX Protocol↵
· symbol: vix↵
- id:·0x↵
· name:·0x Protocol↵
· symbol: zrx↵
- id:·0x0-ai-ai-smart-contract↵
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id:·0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 245,044chars_og/tok: 6.6encoded_chars/tok: 2.5
[19332]{id,symbol,name}:↵
· _,gib,༼·つ ��_◕ ���つ↵
··000-capital,"000",000 Capital↵
··01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
··01-token,"01","01"↵
··0chain,zcn,Zus↵
··0vix-protocol,vix,0VIX Protocol↵
··0x,zrx,0x Protocol↵
··0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
··0x678-landwolf-1933,wolf,Landwolf↵
··0xgasless-2,0xgas,0xGasless↵
··0xgen,xgn,0xGen↵
··0x-leverage,oxl,0x Leverage↵
··0xlsd,0xlsd,0xLSD↵
··0xmonk,monk,0xMonk by Virtuals↵
··0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
··0xshadow,0xs,0xShadow↵
··0xsim-by-virtuals,sage,0xsim by Virtuals↵
··0xy,0xy,0xy↵
· "-10",loong,龙↵
··1000bonk,1000bonk,1000BONK↵
··1000btt,1000btt,1000BTT↵
··1000cat,1000cat,1000CAT↵
··1000chems,1000cheems,1000CHEMS↵
··1000mog,1000mog,1000MOG
... (truncated)
· _,gib,༼·つ ��_◕ ���つ↵
··000-capital,"000",000 Capital↵
··01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
··01-token,"01","01"↵
··0chain,zcn,Zus↵
··0vix-protocol,vix,0VIX Protocol↵
··0x,zrx,0x Protocol↵
··0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
··0x678-landwolf-1933,wolf,Landwolf↵
··0xgasless-2,0xgas,0xGasless↵
··0xgen,xgn,0xGen↵
··0x-leverage,oxl,0x Leverage↵
··0xlsd,0xlsd,0xLSD↵
··0xmonk,monk,0xMonk by Virtuals↵
··0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
··0xshadow,0xs,0xShadow↵
··0xsim-by-virtuals,sage,0xsim by Virtuals↵
··0xy,0xy,0xy↵
· "-10",loong,龙↵
··1000bonk,1000bonk,1000BONK↵
··1000btt,1000btt,1000BTT↵
··1000cat,1000cat,1000CAT↵
··1000chems,1000cheems,1000CHEMS↵
··1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 232,225chars_og/tok: 6.9encoded_chars/tok: 2.6
{@id,symbol,name#19332|_,gib,"༼·つ ��_◕ ���つ"|000-capital,"000","000 Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIX Protocol"|0x,zrx,"0x Protocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0x Leverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonk by Virtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"Avalanche Bridged ZRX (Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsim by Virtuals"|0xy,0xy,0xy|"-10",loong,龙|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS (Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000x by Virtuals"|100-token,100¥,100¥|100xdarren,100x,100xDarren|10-figs,figs,"10 figs"|"-11",赵长娥,赵长娥|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1 Coin Can Change Your Life"|1-community-can-change-your-life,community,"1 community can change your life"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1 dog can change your life"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1Hive Water"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCH yVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1Move Token"|1-narrative-can-change-your-life,narrative,"1 narrative can change your life"|1-one,one,"1 (one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUS DAO"|1-squirrel,peanut,"OG Peanut"|1-token,"1","1"|2004-pepe,bog,"2004 PEPE"|2025-token,"2025","2025 TOKEN"|2077-code,"2077","2077 CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23 Turtles"|2-3-years-and-forget,"23","2-3 years and forget"|24k-gold-pepe,goldpepe,"24K Gold PEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2G Carbon Coin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"Meow Meow Coin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACE ART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404—GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47th POTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4 Next Unicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TB Coin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-Way Mirror Money"|"-5",�����,�������������|500m-piece-of-paper,paper,"$500M piece of paper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5th Scape"|"-6"," "," "|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",∅,Voidify|717ai-by-virtuals,wire,"717ai by Virtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",🔶,🔶|8004-dog,dog8004,"8004 Dog"|8008-token,"8008","8008"|888coin,發發發,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8 Ball"|8-bit-coin,coin,"8-Bit Coin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",∑,∑|9-5,9-5,9to5|99-bitcoins,99btc,"99 Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16z AI Dog"|a51-finance,a51,"A51 Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaa cat"|aaai_agent-by-virtuals,aaai,"AAAI_agent by Virtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADex Finance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"Aark Digital"|aarna-afi-802v2,"afi·802v2","aarna afi·802v2"|aarna-atv111,atv111,"aarna atv111"|aarna-atv111-arbitrum,atv111,"aarna atv111 (Arbitrum)"|aarna-atv111-sonic,atv111,"aarna atv111 (Sonic)"|aarna-atv-808,atv808,"aarna atv·808"|aarna-atv-usdc,atvusdc,"aarna atv USDC (Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarna atv USDC (Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"Aave AAVE"|aave-amm-bptbalweth,aammbptbalweth,"Aave AMM BptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"Aave AMM BptWBTCWETH"|aave-amm-dai,aammdai,"Aave AMM DAI"|aave-amm-uniaaveweth,aammuniaaveweth,"Aave AMM UniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"Aave AMM UniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"Aave AMM UniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"Aave AMM UniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"Aave AMM UniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"Aave AMM UniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"Aave AMM·
... (truncated)
... (truncated)
minemizer
chars: 622,925tokens: 237,950chars_og/tok: 6.8encoded_chars/tok: 2.6
id; symbol; name↵
_; gib; ���·つ ��_◕ ���つ↵
000-capital;·000;·000 Capital↵
01111010011110000110001001110100-token;·01111010011110000110001001110100;·01111010011110000110001001110100↵
01-token;·01;·01↵
0chain; zcn; Zus↵
0vix-protocol; vix;·0VIX Protocol↵
0x; zrx;·0x Protocol↵
0x0-ai-ai-smart-contract;·0x0;·0x0.ai: AI Smart Contract↵
0x678-landwolf-1933; wolf; Landwolf↵
0xgasless-2;·0xgas;·0xGasless↵
0xgen; xgn;·0xGen↵
0x-leverage; oxl;·0x Leverage↵
0xlsd;·0xlsd;·0xLSD↵
0xmonk; monk;·0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow;·0xs;·0xShadow↵
0xsim-by-virtuals; sage;·0xsim by Virtuals↵
0xy;·0xy;·0xy↵
-10; loong; ���↵
1000bonk;·1000bonk;·1000BONK↵
1000btt;·1000btt;·1000BTT↵
1000cat;·1000cat;·1000CAT↵
1000chems;·1000cheems;·1000CHEMS↵
1000mog;·1000mog;·1000MOG
... (truncated)
_; gib; ���·つ ��_◕ ���つ↵
000-capital;·000;·000 Capital↵
01111010011110000110001001110100-token;·01111010011110000110001001110100;·01111010011110000110001001110100↵
01-token;·01;·01↵
0chain; zcn; Zus↵
0vix-protocol; vix;·0VIX Protocol↵
0x; zrx;·0x Protocol↵
0x0-ai-ai-smart-contract;·0x0;·0x0.ai: AI Smart Contract↵
0x678-landwolf-1933; wolf; Landwolf↵
0xgasless-2;·0xgas;·0xGasless↵
0xgen; xgn;·0xGen↵
0x-leverage; oxl;·0x Leverage↵
0xlsd;·0xlsd;·0xLSD↵
0xmonk; monk;·0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow;·0xs;·0xShadow↵
0xsim-by-virtuals; sage;·0xsim by Virtuals↵
0xy;·0xy;·0xy↵
-10; loong; ���↵
1000bonk;·1000bonk;·1000BONK↵
1000btt;·1000btt;·1000BTT↵
1000cat;·1000cat;·1000CAT↵
1000chems;·1000cheems;·1000CHEMS↵
1000mog;·1000mog;·1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 237,554chars_og/tok: 6.8encoded_chars/tok: 2.5
id;symbol;name↵
_;gib;༼·つ ��_◕ ���つ↵
000-capital;000;000 Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIX Protocol↵
0x;zrx;0x Protocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0x Leverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsim by Virtuals↵
0xy;0xy;0xy↵
-10;loong;龙↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
_;gib;༼·つ ��_◕ ���つ↵
000-capital;000;000 Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIX Protocol↵
0x;zrx;0x Protocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0x Leverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsim by Virtuals↵
0xy;0xy;0xy↵
-10;loong;龙↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: qwen2.5 (Qwen/Qwen2.5-0.5B)
JSON (pretty)
chars: 1,320tokens: 455chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id":·2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
· {↵
··· "id":·1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id":·2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
JSON (min)
chars: 760tokens: 246chars_og/tok: 5.4encoded_chars/tok: 3.1
[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 306chars_og/tok: 4.3encoded_chars/tok: 2.7
- id:·1↵
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC↵
····· country: USA↵
··· name: Grace↵
· tags:↵
· - admin↵
· - verified↵
- id:·2↵
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London↵
····· country: UK↵
··· name: Henry↵
· tags:↵
· - user↵
- id:·3↵
· metadata:↵
··· created: '2024-01-05'
... (truncated)
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC↵
····· country: USA↵
··· name: Grace↵
· tags:↵
· - admin↵
· - verified↵
- id:·2↵
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London↵
····· country: UK↵
··· name: Henry↵
· tags:↵
· - user↵
- id:·3↵
· metadata:↵
··· created: '2024-01-05'
... (truncated)
TOON
chars: 881tokens: 304chars_og/tok: 4.3encoded_chars/tok: 2.9
[5]:↵
· - id:·1↵
··· profile:↵
····· name: Grace↵
····· location:↵
······· city: NYC↵
······· country: USA↵
··· tags[2]: admin,verified↵
··· metadata:↵
····· created:·2024-01-15↵
· - id:·2↵
··· profile:↵
····· name: Henry↵
····· location:↵
······· city: London↵
······· country: UK↵
··· tags[1]: user↵
··· metadata:↵
····· created:·2024-02-20↵
····· updated:·2024-03-10↵
· - id:·3↵
··· profile:↵
····· name: Ivy↵
····· location:↵
······· city: Tokyo
... (truncated)
· - id:·1↵
··· profile:↵
····· name: Grace↵
····· location:↵
······· city: NYC↵
······· country: USA↵
··· tags[2]: admin,verified↵
··· metadata:↵
····· created:·2024-01-15↵
· - id:·2↵
··· profile:↵
····· name: Henry↵
····· location:↵
······· city: London↵
······· country: UK↵
··· tags[1]: user↵
··· metadata:↵
····· created:·2024-02-20↵
····· updated:·2024-03-10↵
· - id:·3↵
··· profile:↵
····· name: Ivy↵
····· location:↵
······· city: Tokyo
... (truncated)
TSON
chars: 453tokens: 203chars_og/tok: 6.5encoded_chars/tok: 2.2
{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 191chars_og/tok: 6.9encoded_chars/tok: 2.2
id; profile{ name; location{ city; country}}; tags[]; metadata{ created; ...}↵
1; { Grace; { NYC; USA}}; [ admin; verified]; {·2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; {·2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; {·2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; {·2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; {·2023-12-01; updated:2024-02-15}
1; { Grace; { NYC; USA}}; [ admin; verified]; {·2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; {·2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; {·2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; {·2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; {·2023-12-01; updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 191chars_og/tok: 6.9encoded_chars/tok: 1.9
id;profile{ name;location{ city;country}};tags[];metadata{ created;...}↵
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
simple_flat.json — Original size: 763 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 763tokens: 264chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "role": "Engineer",↵
··· "department": "Backend"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "role": "Designer",↵
··· "department": "Frontend"↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "role": "Manager",↵
··· "department": "Product"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "David",↵
··· "role": "Engineer",↵
··· "department": "Infrastructure"↵
· },
... (truncated)
JSON (min)
chars: 522tokens: 137chars_og/tok: 5.6encoded_chars/tok: 3.8
[{"id":1,"name":"Alice","role":"Engineer","department":"Backend"},{"id":2,"name":"Bob","role":"Designer","department":"Frontend"},{"id":3,"name":"Carol","role":"Manager","department":"Product"},{"id":4,"name":"David","role":"Engineer","department":"Infrastructure"},{"id":5,"name":"Eva","role":"Analyst","department":"Data"},{"id":6,"name":"Frank","role":"Engineer","department":"Backend"},{"id":7,"name":"Grace","role":"Designer","department":"Mobile"},{"id":8,"name":"Henry","role":"Manager","department":"Engineering"}]
CSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id,name,role,department
↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
1,Alice,Engineer,Backend ↵
2,Bob,Designer,Frontend ↵
3,Carol,Manager,Product ↵
4,David,Engineer,Infrastructure ↵
5,Eva,Analyst,Data ↵
6,Frank,Engineer,Backend ↵
7,Grace,Designer,Mobile ↵
8,Henry,Manager,Engineering ↵
TSV
chars: 234tokens: 77chars_og/tok: 9.9encoded_chars/tok: 3.0
id name→role→department
↵
1 Alice→Engineer→Backend ↵
2 Bob→Designer Frontend ↵
3 Carol→Manager→Product ↵
4 David→Engineer→Infrastructure ↵
5 Eva→Analyst Data ↵
6 Frank→Engineer→Backend ↵
7→Grace→Designer Mobile ↵
8 Henry→Manager→Engineering ↵
1 Alice→Engineer→Backend ↵
2 Bob→Designer Frontend ↵
3 Carol→Manager→Product ↵
4 David→Engineer→Infrastructure ↵
5 Eva→Analyst Data ↵
6 Frank→Engineer→Backend ↵
7→Grace→Designer Mobile ↵
8 Henry→Manager→Engineering ↵
YAML
chars: 489tokens: 169chars_og/tok: 4.5encoded_chars/tok: 2.9
- department: Backend↵
· id:·1↵
· name: Alice↵
· role: Engineer↵
- department: Frontend↵
· id:·2↵
· name: Bob↵
· role: Designer↵
- department: Product↵
· id:·3↵
· name: Carol↵
· role: Manager↵
- department: Infrastructure↵
· id:·4↵
· name: David↵
· role: Engineer↵
- department: Data↵
· id:·5↵
· name: Eva↵
· role: Analyst↵
- department: Backend↵
· id:·6↵
· name: Frank↵
· role: Engineer↵
- department: Mobile
... (truncated)
· id:·1↵
· name: Alice↵
· role: Engineer↵
- department: Frontend↵
· id:·2↵
· name: Bob↵
· role: Designer↵
- department: Product↵
· id:·3↵
· name: Carol↵
· role: Manager↵
- department: Infrastructure↵
· id:·4↵
· name: David↵
· role: Engineer↵
- department: Data↵
· id:·5↵
· name: Eva↵
· role: Analyst↵
- department: Backend↵
· id:·6↵
· name: Frank↵
· role: Engineer↵
- department: Mobile
... (truncated)
TOON
chars: 246tokens: 96chars_og/tok: 7.9encoded_chars/tok: 2.6
[8]{id,name,role,department}:↵
··1,Alice,Engineer,Backend↵
··2,Bob,Designer,Frontend↵
··3,Carol,Manager,Product↵
··4,David,Engineer,Infrastructure↵
··5,Eva,Analyst,Data↵
··6,Frank,Engineer,Backend↵
··7,Grace,Designer,Mobile↵
··8,Henry,Manager,Engineering
··1,Alice,Engineer,Backend↵
··2,Bob,Designer,Frontend↵
··3,Carol,Manager,Product↵
··4,David,Engineer,Infrastructure↵
··5,Eva,Analyst,Data↵
··6,Frank,Engineer,Backend↵
··7,Grace,Designer,Mobile↵
··8,Henry,Manager,Engineering
TSON
chars: 229tokens: 80chars_og/tok: 9.5encoded_chars/tok: 2.9
{@id,name,role,department#8|1,Alice,Engineer,Backend|2,Bob,Designer,Frontend|3,Carol,Manager,Product|4,David,Engineer,Infrastructure|5,Eva,Analyst,Data|6,Frank,Engineer,Backend|7,Grace,Designer,Mobile|8,Henry,Manager,Engineering}
minemizer
chars: 251tokens: 72chars_og/tok: 10.6encoded_chars/tok: 3.5
id; name; role; department↵
1; Alice; Engineer; Backend↵
2; Bob; Designer; Frontend↵
3; Carol; Manager; Product↵
4; David; Engineer; Infrastructure↵
5; Eva; Analyst; Data↵
6; Frank; Engineer; Backend↵
7; Grace; Designer; Mobile↵
8; Henry; Manager; Engineering
1; Alice; Engineer; Backend↵
2; Bob; Designer; Frontend↵
3; Carol; Manager; Product↵
4; David; Engineer; Infrastructure↵
5; Eva; Analyst; Data↵
6; Frank; Engineer; Backend↵
7; Grace; Designer; Mobile↵
8; Henry; Manager; Engineering
minemizer (compact)
chars: 224tokens: 77chars_og/tok: 9.9encoded_chars/tok: 2.9
id;name;role;department↵
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
1;Alice;Engineer;Backend↵
2;Bob;Designer;Frontend↵
3;Carol;Manager;Product↵
4;David;Engineer;Infrastructure↵
5;Eva;Analyst;Data↵
6;Frank;Engineer;Backend↵
7;Grace;Designer;Mobile↵
8;Henry;Manager;Engineering
nested_objects.json — Original size: 741 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 741tokens: 252chars_og/tok: 2.9encoded_chars/tok: 2.9
[↵
· {↵
··· "id":·1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id":·2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id":·3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
· {↵
··· "id":·1,↵
··· "user": {↵
····· "name": "Alice",↵
····· "email": "alice@example.com"↵
··· },↵
··· "status": "active"↵
· },↵
· {↵
··· "id":·2,↵
··· "user": {↵
····· "name": "Bob",↵
····· "email": "bob@example.com"↵
··· },↵
··· "status": "inactive"↵
· },↵
· {↵
··· "id":·3,↵
··· "user": {↵
····· "name": "Carol",↵
····· "email": "carol@example.com"↵
··· },↵
··· "status": "active"↵
· },
... (truncated)
JSON (min)
chars: 470tokens: 127chars_og/tok: 5.8encoded_chars/tok: 3.7
[{"id":1,"user":{"name":"Alice","email":"alice@example.com"},"status":"active"},{"id":2,"user":{"name":"Bob","email":"bob@example.com"},"status":"inactive"},{"id":3,"user":{"name":"Carol","email":"carol@example.com"},"status":"active"},{"id":4,"user":{"name":"David","email":"david@example.com"},"status":"pending"},{"id":5,"user":{"name":"Eva","email":"eva@example.com"},"status":"active"},{"id":6,"user":{"name":"Frank","email":"frank@example.com"},"status":"active"}]
CSV: N/A
TSV: N/A
YAML
chars: 463tokens: 158chars_og/tok: 4.7encoded_chars/tok: 2.9
- id:·1↵
· status: active↵
· user:↵
··· email: alice@example.com↵
··· name: Alice↵
- id:·2↵
· status: inactive↵
· user:↵
··· email: bob@example.com↵
··· name: Bob↵
- id:·3↵
· status: active↵
· user:↵
··· email: carol@example.com↵
··· name: Carol↵
- id:·4↵
· status: pending↵
· user:↵
··· email: david@example.com↵
··· name: David↵
- id:·5↵
· status: active↵
· user:↵
··· email: eva@example.com↵
··· name: Eva
... (truncated)
· status: active↵
· user:↵
··· email: alice@example.com↵
··· name: Alice↵
- id:·2↵
· status: inactive↵
· user:↵
··· email: bob@example.com↵
··· name: Bob↵
- id:·3↵
· status: active↵
· user:↵
··· email: carol@example.com↵
··· name: Carol↵
- id:·4↵
· status: pending↵
· user:↵
··· email: david@example.com↵
··· name: David↵
- id:·5↵
· status: active↵
· user:↵
··· email: eva@example.com↵
··· name: Eva
... (truncated)
TOON
chars: 527tokens: 166chars_og/tok: 4.5encoded_chars/tok: 3.2
[6]:↵
· - id:·1↵
··· user:↵
····· name: Alice↵
····· email: alice@example.com↵
··· status: active↵
· - id:·2↵
··· user:↵
····· name: Bob↵
····· email: bob@example.com↵
··· status: inactive↵
· - id:·3↵
··· user:↵
····· name: Carol↵
····· email: carol@example.com↵
··· status: active↵
· - id:·4↵
··· user:↵
····· name: David↵
····· email: david@example.com↵
··· status: pending↵
· - id:·5↵
··· user:↵
····· name: Eva↵
····· email: eva@example.com
... (truncated)
· - id:·1↵
··· user:↵
····· name: Alice↵
····· email: alice@example.com↵
··· status: active↵
· - id:·2↵
··· user:↵
····· name: Bob↵
····· email: bob@example.com↵
··· status: inactive↵
· - id:·3↵
··· user:↵
····· name: Carol↵
····· email: carol@example.com↵
··· status: active↵
· - id:·4↵
··· user:↵
····· name: David↵
····· email: david@example.com↵
··· status: pending↵
· - id:·5↵
··· user:↵
····· name: Eva↵
····· email: eva@example.com
... (truncated)
TSON
chars: 249tokens: 75chars_og/tok: 9.9encoded_chars/tok: 3.3
{@id,user(@name,email),status#6|1,{Alice,"alice@example.com"},active|2,{Bob,"bob@example.com"},inactive|3,{Carol,"carol@example.com"},active|4,{David,"david@example.com"},pending|5,{Eva,"eva@example.com"},active|6,{Frank,"frank@example.com"},active}
minemizer
chars: 259tokens: 77chars_og/tok: 9.6encoded_chars/tok: 3.4
id; user{ name; email}; status↵
1; { Alice; alice@example.com}; active↵
2; { Bob; bob@example.com}; inactive↵
3; { Carol; carol@example.com}; active↵
4; { David; david@example.com}; pending↵
5; { Eva; eva@example.com}; active↵
6; { Frank; frank@example.com}; active
1; { Alice; alice@example.com}; active↵
2; { Bob; bob@example.com}; inactive↵
3; { Carol; carol@example.com}; active↵
4; { David; david@example.com}; pending↵
5; { Eva; eva@example.com}; active↵
6; { Frank; frank@example.com}; active
minemizer (compact)
chars: 232tokens: 78chars_og/tok: 9.5encoded_chars/tok: 3.0
id;user{ name;email};status↵
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
1;{Alice;alice@example.com};active↵
2;{Bob;bob@example.com};inactive↵
3;{Carol;carol@example.com};active↵
4;{David;david@example.com};pending↵
5;{Eva;eva@example.com};active↵
6;{Frank;frank@example.com};active
lists_of_primitives.json — Original size: 610 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 610tokens: 217chars_og/tok: 2.8encoded_chars/tok: 2.8
[↵
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
· {↵
··· "id":·1,↵
··· "name": "Alice",↵
··· "skills": [↵
····· "python",↵
····· "go",↵
····· "rust"↵
··· ]↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Bob",↵
··· "skills": [↵
····· "javascript",↵
····· "typescript"↵
··· ]↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Carol",↵
··· "skills": [↵
····· "java",↵
····· "kotlin",↵
····· "scala",
... (truncated)
JSON (min)
chars: 330tokens: 103chars_og/tok: 5.9encoded_chars/tok: 3.2
[{"id":1,"name":"Alice","skills":["python","go","rust"]},{"id":2,"name":"Bob","skills":["javascript","typescript"]},{"id":3,"name":"Carol","skills":["java","kotlin","scala","groovy"]},{"id":4,"name":"David","skills":["c","cpp"]},{"id":5,"name":"Eva","skills":["ruby","elixir","erlang"]},{"id":6,"name":"Frank","skills":["swift"]}]
CSV: N/A
TSV: N/A
YAML
chars: 341tokens: 149chars_og/tok: 4.1encoded_chars/tok: 2.3
- id:·1↵
· name: Alice↵
· skills:↵
· - python↵
· - go↵
· - rust↵
- id:·2↵
· name: Bob↵
· skills:↵
· - javascript↵
· - typescript↵
- id:·3↵
· name: Carol↵
· skills:↵
· - java↵
· - kotlin↵
· - scala↵
· - groovy↵
- id:·4↵
· name: David↵
· skills:↵
· - c↵
· - cpp↵
- id:·5↵
· name: Eva
... (truncated)
· name: Alice↵
· skills:↵
· - python↵
· - go↵
· - rust↵
- id:·2↵
· name: Bob↵
· skills:↵
· - javascript↵
· - typescript↵
- id:·3↵
· name: Carol↵
· skills:↵
· - java↵
· - kotlin↵
· - scala↵
· - groovy↵
- id:·4↵
· name: David↵
· skills:↵
· - c↵
· - cpp↵
- id:·5↵
· name: Eva
... (truncated)
TOON
chars: 339tokens: 137chars_og/tok: 4.5encoded_chars/tok: 2.5
[6]:↵
· - id:·1↵
··· name: Alice↵
··· skills[3]: python,go,rust↵
· - id:·2↵
··· name: Bob↵
··· skills[2]: javascript,typescript↵
· - id:·3↵
··· name: Carol↵
··· skills[4]: java,kotlin,scala,groovy↵
· - id:·4↵
··· name: David↵
··· skills[2]: c,cpp↵
· - id:·5↵
··· name: Eva↵
··· skills[3]: ruby,elixir,erlang↵
· - id:·6↵
··· name: Frank↵
··· skills[1]: swift
· - id:·1↵
··· name: Alice↵
··· skills[3]: python,go,rust↵
· - id:·2↵
··· name: Bob↵
··· skills[2]: javascript,typescript↵
· - id:·3↵
··· name: Carol↵
··· skills[4]: java,kotlin,scala,groovy↵
· - id:·4↵
··· name: David↵
··· skills[2]: c,cpp↵
· - id:·5↵
··· name: Eva↵
··· skills[3]: ruby,elixir,erlang↵
· - id:·6↵
··· name: Frank↵
··· skills[1]: swift
TSON
chars: 168tokens: 65chars_og/tok: 9.4encoded_chars/tok: 2.6
{@id,name,skills#6|1,Alice,[python,go,rust]|2,Bob,[javascript,typescript]|3,Carol,[java,kotlin,scala,groovy]|4,David,[c,cpp]|5,Eva,[ruby,elixir,erlang]|6,Frank,[swift]}
minemizer
chars: 194tokens: 71chars_og/tok: 8.6encoded_chars/tok: 2.7
id; name; skills[]↵
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
1; Alice; [ python; go; rust]↵
2; Bob; [ javascript; typescript]↵
3; Carol; [ java; kotlin; scala; groovy]↵
4; David; [ c; cpp]↵
5; Eva; [ ruby; elixir; erlang]↵
6; Frank; [ swift]
minemizer (compact)
chars: 165tokens: 70chars_og/tok: 8.7encoded_chars/tok: 2.4
id;name;skills[]↵
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
1;Alice;[python;go;rust]↵
2;Bob;[javascript;typescript]↵
3;Carol;[java;kotlin;scala;groovy]↵
4;David;[c;cpp]↵
5;Eva;[ruby;elixir;erlang]↵
6;Frank;[swift]
sparse_data.json — Original size: 589 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 589tokens: 224chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id":·1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Dave",↵
··· "remote": true↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id":·5,↵
··· "name": "Grace",
... (truncated)
· {↵
··· "id":·1,↵
··· "name": "Carol",↵
··· "role": "Manager"↵
· },↵
· {↵
··· "id":·2,↵
··· "name": "Dave",↵
··· "remote": true↵
· },↵
· {↵
··· "id":·3,↵
··· "name": "Eve",↵
··· "role": "Designer",↵
··· "team": "UX"↵
· },↵
· {↵
··· "id":·4,↵
··· "name": "Frank",↵
··· "department": "Engineering"↵
· },↵
· {↵
··· "id":·5,↵
··· "name": "Grace",
... (truncated)
JSON (min)
chars: 378tokens: 114chars_og/tok: 5.2encoded_chars/tok: 3.3
[{"id":1,"name":"Carol","role":"Manager"},{"id":2,"name":"Dave","remote":true},{"id":3,"name":"Eve","role":"Designer","team":"UX"},{"id":4,"name":"Frank","department":"Engineering"},{"id":5,"name":"Grace","role":"Engineer","remote":true,"team":"Platform"},{"id":6,"name":"Henry","role":"Analyst"},{"id":7,"name":"Ivy"},{"id":8,"name":"Jack","department":"Sales","remote":false}]
CSV: N/A
TSV: N/A
YAML
chars: 356tokens: 143chars_og/tok: 4.1encoded_chars/tok: 2.5
- id:·1↵
· name: Carol↵
· role: Manager↵
- id:·2↵
· name: Dave↵
· remote: true↵
- id:·3↵
· name: Eve↵
· role: Designer↵
· team: UX↵
- department: Engineering↵
· id:·4↵
· name: Frank↵
- id:·5↵
· name: Grace↵
· remote: true↵
· role: Engineer↵
· team: Platform↵
- id:·6↵
· name: Henry↵
· role: Analyst↵
- id:·7↵
· name: Ivy↵
- department: Sales↵
· id:·8
... (truncated)
· name: Carol↵
· role: Manager↵
- id:·2↵
· name: Dave↵
· remote: true↵
- id:·3↵
· name: Eve↵
· role: Designer↵
· team: UX↵
- department: Engineering↵
· id:·4↵
· name: Frank↵
- id:·5↵
· name: Grace↵
· remote: true↵
· role: Engineer↵
· team: Platform↵
- id:·6↵
· name: Henry↵
· role: Analyst↵
- id:·7↵
· name: Ivy↵
- department: Sales↵
· id:·8
... (truncated)
TOON
chars: 414tokens: 153chars_og/tok: 3.8encoded_chars/tok: 2.7
[8]:↵
· - id:·1↵
··· name: Carol↵
··· role: Manager↵
· - id:·2↵
··· name: Dave↵
··· remote: true↵
· - id:·3↵
··· name: Eve↵
··· role: Designer↵
··· team: UX↵
· - id:·4↵
··· name: Frank↵
··· department: Engineering↵
· - id:·5↵
··· name: Grace↵
··· role: Engineer↵
··· remote: true↵
··· team: Platform↵
· - id:·6↵
··· name: Henry↵
··· role: Analyst↵
· - id:·7↵
··· name: Ivy↵
· - id:·8
... (truncated)
· - id:·1↵
··· name: Carol↵
··· role: Manager↵
· - id:·2↵
··· name: Dave↵
··· remote: true↵
· - id:·3↵
··· name: Eve↵
··· role: Designer↵
··· team: UX↵
· - id:·4↵
··· name: Frank↵
··· department: Engineering↵
· - id:·5↵
··· name: Grace↵
··· role: Engineer↵
··· remote: true↵
··· team: Platform↵
· - id:·6↵
··· name: Henry↵
··· role: Analyst↵
· - id:·7↵
··· name: Ivy↵
· - id:·8
... (truncated)
TSON
chars: 300tokens: 109chars_og/tok: 5.4encoded_chars/tok: 2.8
[{@id,name,role|1,Carol,Manager},{@id,name,remote|2,Dave,true},{@id,name,role,team|3,Eve,Designer,UX},{@id,name,department|4,Frank,Engineering},{@id,name,role,remote,team|5,Grace,Engineer,true,Platform},{@id,name,role|6,Henry,Analyst},{@id,name|7,Ivy},{@id,name,department,remote|8,Jack,Sales,false}]
minemizer
chars: 232tokens: 77chars_og/tok: 7.6encoded_chars/tok: 3.0
id; name; role↵
1; Carol; Manager↵
2; Dave; ; remote:True↵
3; Eve; Designer; team:UX↵
4; Frank; ; department:Engineering↵
5; Grace; Engineer; remote:True; team:Platform↵
6; Henry; Analyst↵
7; Ivy; ↵
8; Jack; ; department:Sales; remote:False
1; Carol; Manager↵
2; Dave; ; remote:True↵
3; Eve; Designer; team:UX↵
4; Frank; ; department:Engineering↵
5; Grace; Engineer; remote:True; team:Platform↵
6; Henry; Analyst↵
7; Ivy; ↵
8; Jack; ; department:Sales; remote:False
minemizer (compact)
chars: 207tokens: 77chars_og/tok: 7.6encoded_chars/tok: 2.7
id;name;role↵
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
1;Carol;Manager↵
2;Dave;;remote:True↵
3;Eve;Designer;team:UX↵
4;Frank;;department:Engineering↵
5;Grace;Engineer;remote:True;team:Platform↵
6;Henry;Analyst↵
7;Ivy;↵
8;Jack;;department:Sales;remote:False
coingecko_coins.json — Original size: 1611780 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 1,611,780tokens: 609,227chars_og/tok: 2.6encoded_chars/tok: 2.6
[↵
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
· {↵
··· "id": "_",↵
··· "symbol": "gib",↵
··· "name": "\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"↵
· },↵
· {↵
··· "id": "000-capital",↵
··· "symbol": "000",↵
··· "name": "000 Capital"↵
· },↵
· {↵
··· "id": "01111010011110000110001001110100-token",↵
··· "symbol": "01111010011110000110001001110100",↵
··· "name": "01111010011110000110001001110100"↵
· },↵
· {↵
··· "id": "01-token",↵
··· "symbol": "01",↵
··· "name": "01"↵
· },↵
· {↵
··· "id": "0chain",↵
··· "symbol": "zcn",↵
··· "name": "Zus"
... (truncated)
JSON (min)
chars: 1,147,811tokens: 359,549chars_og/tok: 4.5encoded_chars/tok: 3.2
[{"id":"_","symbol":"gib","name":"\u0f3c \u3064 \u25d5_\u25d5 \u0f3d\u3064"},{"id":"000-capital","symbol":"000","name":"000 Capital"},{"id":"01111010011110000110001001110100-token","symbol":"01111010011110000110001001110100","name":"01111010011110000110001001110100"},{"id":"01-token","symbol":"01","name":"01"},{"id":"0chain","symbol":"zcn","name":"Zus"},{"id":"0vix-protocol","symbol":"vix","name":"0VIX Protocol"},{"id":"0x","symbol":"zrx","name":"0x Protocol"},{"id":"0x0-ai-ai-smart-contract","symbol":"0x0","name":"0x0.ai: AI Smart Contract"},{"id":"0x678-landwolf-1933","symbol":"wolf","name":"Landwolf"},{"id":"0xgasless-2","symbol":"0xgas","name":"0xGasless"},{"id":"0xgen","symbol":"xgn","name":"0xGen"},{"id":"0x-leverage","symbol":"oxl","name":"0x Leverage"},{"id":"0xlsd","symbol":"0xlsd","name":"0xLSD"},{"id":"0xmonk","symbol":"monk","name":"0xMonk by Virtuals"},{"id":"0x-protocol-avalanche-bridged-zrx-e","symbol":"zrx.e","name":"Avalanche Bridged ZRX (Avalanche)"},{"id":"0xshadow","symbol":"0xs","name":"0xShadow"},{"id":"0xsim-by-virtuals","symbol":"sage","name":"0xsim by Virtuals"},{"id":"0xy","symbol":"0xy","name":"0xy"},{"id":"-10","symbol":"loong","name":"\u9f99"},{"id":"1000bonk","symbol":"1000bonk","name":"1000BONK"},{"id":"1000btt","symbol":"1000btt","name":"1000BTT"},{"id":"1000cat","symbol":"1000cat","name":"1000CAT"},{"id":"1000chems","symbol":"1000cheems","name":"1000CHEMS"},{"id":"1000mog","symbol":"1000mog","name":"1000MOG"},{"id":"1000rats","symbol":"1000rats","name":"1000RATS"},{"id":"1000sats-ordinals","symbol":"1000sats","name":"1000SATS (Ordinals)"},{"id":"1000shib","symbol":"1000shib","name":"1000SHIB"},{"id":"1000x-by-virtuals","symbol":"1000x","name":"1000x by Virtuals"},{"id":"100-token","symbol":"100\u00a5","name":"100\u00a5"},{"id":"100xdarren","symbol":"100x","name":"100xDarren"},{"id":"10-figs","symbol":"figs","name":"10 figs"},{"id":"-11","symbol":"\u8d75\u957f\u5a25","name":"\u8d75\u957f\u5a25"},{"id":"11am","symbol":"11am","name":"11am"},{"id":"1984-token","symbol":"1984","name":"1984"},{"id":"1art","symbol":"1art","name":"OneArt"},{"id":"1-coin-can-change-your-life","symbol":"1-coin-can-change-your-life","name":"1 Coin Can Change Your Life"},{"id":"1-community-can-change-your-life","symbol":"community","name":"1 community can change your life"},{"id":"1dev","symbol":"1dev","name":"1DEV"},{"id":"1-dog-can-change-your-life","symbol":"1dog","name":"1 dog can change your life"},{"id":"1-dollar-sol-coin","symbol":"$1","name":"$1"},{"id":"1guy","symbol":"1guy","name":"1GUY"},{"id":"1hive-water","symbol":"water","name":"1Hive Water"},{"id":"1hub-ai","symbol":"1hub","name":"1Hub.ai"},{"id":"1inch","symbol":"1inch","name":"1INCH"},{"id":"1inch-yvault","symbol":"yv1inch","name":"1INCH yVault"},{"id":"1intro","symbol":"chef","name":"CoinChef"},{"id":"1mbabydoge","symbol":"1mbabydoge","name":"1MBABYDOGE"},{"id":"1million-nfts","symbol":"1mil","name":"1MillionNFTs"},{"id":"1move-token","symbol":"1mt","name":"1Move Token"},{"id":"1-narrative-can-change-your-life","symbol":"narrative","name":"1 narrative can change your life"},{"id":"1-one","symbol":"one","name":"1 (one)"},{"id":"1-percent","symbol":"1%","name":"1%"},{"id":"1rus-btc25","symbol":"@btc25","name":"@BTC25"},{"id":"1rus-dao","symbol":"1rusd","name":"1RUS DAO"},{"id":"1-squirrel","symbol":"peanut","name":"OG Peanut"},{"id":"1-token","symbol":"1","name":"1"},{"id":"2004-pepe","symbol":"bog","name":"2004 PEPE"},{"id":"2025-token","symbol":"2025","name":"2025 TOKEN"},{"id":"2077-code","symbol":"2077","name":"2077 CODE"},{"id":"2080","symbol":"2080","name":"2080"},{"id":"21million","symbol":"21m","name":"21Million"},{"id":"23-turtles","symbol":"ai23t","name":"23 Turtles"},{"id":"2-3-years-and-forget","symbol":"23","name":"2-3 years and forget"},{"id":"24k-gold-pepe","symbol":"goldpepe","name":"24K Gold PEPE"},{"id":"2dai-io","symbol":"2dai","name":"2DAI.io[Old]"},{"id":"2dai-io-2","symbol":"2dai","name":"2DAI.io"},{"id":"2g-carbon-coin","symbol":"2gcc","name":"2G Carbon Coin"},{"id":"2moon","symbol":"moon","name":"2MOON"},{"id":"2-token","symbol":"2","name":"2"},{"id":"-3","symbol":"meow","name":"Meow Meow Coin"},{"id":"3000-token","symbol":"3000","name":"3000"},{"id":"360noscope420blazeit","symbol":"mlg","name":"360noscope420blazeit"},{"id":"375ai","symbol":"eat","name":"375ai"},{"id":"39a-fun","symbol":"39a","name":"39a.fun"},{"id":"3a-lending-protocol","symbol":"a3a","name":"3A"},{"id":"3bubu","symbol":"3bubu","name":"3BuBu"},{"id":"3dpass","symbol":"p3d","name":"3DPass"},{"id":"3space-art","symbol":"pace","name":"3SPACE ART"},{"id":"4","symbol":"four","name":"4"},{"id":"401jk","symbol":"401jk","name":"401jK"},{"id":"401k","symbol":"401k","name":"401K"},{"id":"404-gen","symbol":"sn17","name":"404\u2014GEN"},{"id":"4-2","symbol":"4","name":"4"},{"id":"42069coin","symbol":"42069coin","name":"42069COIN"},{"id":"4-2-aminoethyl-benzene-1-2-diol","symbol":"dopamine","name":"4-(2-Aminoethyl)benzene-1,2-diol"},{"id":"42-coin","symbol":"
... (truncated)
... (truncated)
CSV
chars: 603,635tokens: 227,103chars_og/tok: 7.1encoded_chars/tok: 2.7
id,symbol,name
↵
_,gib,��·つ ��_�� ���つ ↵
000-capital,000,000 Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIX Protocol ↵
0x,zrx,0x Protocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0x Leverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsim by Virtuals ↵
0xy,0xy,0xy ↵
-10,loong,�� ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
_,gib,��·つ ��_�� ���つ ↵
000-capital,000,000 Capital ↵
01111010011110000110001001110100-token,01111010011110000110001001110100,01111010011110000110001001110100 ↵
01-token,01,01 ↵
0chain,zcn,Zus ↵
0vix-protocol,vix,0VIX Protocol ↵
0x,zrx,0x Protocol ↵
0x0-ai-ai-smart-contract,0x0,0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933,wolf,Landwolf ↵
0xgasless-2,0xgas,0xGasless ↵
0xgen,xgn,0xGen ↵
0x-leverage,oxl,0x Leverage ↵
0xlsd,0xlsd,0xLSD ↵
0xmonk,monk,0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche) ↵
0xshadow,0xs,0xShadow ↵
0xsim-by-virtuals,sage,0xsim by Virtuals ↵
0xy,0xy,0xy ↵
-10,loong,�� ↵
1000bonk,1000bonk,1000BONK ↵
1000btt,1000btt,1000BTT ↵
1000cat,1000cat,1000CAT ↵
1000chems,1000cheems,1000CHEMS ↵
1000mog,1000mog,1000MOG
... (truncated)
TSV
chars: 603,601tokens: 222,774chars_og/tok: 7.2encoded_chars/tok: 2.7
id symbol name
↵
_ gib→��·つ ��_�� ���つ ↵
000-capital→000→000 Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain zcn Zus ↵
0vix-protocol vix→0VIX Protocol ↵
0x zrx→0x Protocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933 wolf Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen xgn→0xGen ↵
0x-leverage→oxl→0x Leverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e zrx.e Avalanche Bridged ZRX (Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals sage→0xsim by Virtuals ↵
0xy→0xy→0xy ↵
-10→loong→�� ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
_ gib→��·つ ��_�� ���つ ↵
000-capital→000→000 Capital ↵
01111010011110000110001001110100-token→01111010011110000110001001110100→01111010011110000110001001110100 ↵
01-token→01→01 ↵
0chain zcn Zus ↵
0vix-protocol vix→0VIX Protocol ↵
0x zrx→0x Protocol ↵
0x0-ai-ai-smart-contract→0x0→0x0.ai: AI Smart Contract ↵
0x678-landwolf-1933 wolf Landwolf ↵
0xgasless-2→0xgas→0xGasless ↵
0xgen xgn→0xGen ↵
0x-leverage→oxl→0x Leverage ↵
0xlsd→0xlsd→0xLSD ↵
0xmonk→monk→0xMonk by Virtuals ↵
0x-protocol-avalanche-bridged-zrx-e zrx.e Avalanche Bridged ZRX (Avalanche) ↵
0xshadow→0xs→0xShadow ↵
0xsim-by-virtuals sage→0xsim by Virtuals ↵
0xy→0xy→0xy ↵
-10→loong→�� ↵
1000bonk→1000bonk→1000BONK ↵
1000btt→1000btt→1000BTT ↵
1000cat→1000cat→1000CAT ↵
1000chems→1000cheems→1000CHEMS ↵
1000mog→1000mog→1000MOG
... (truncated)
YAML
chars: 1,048,449tokens: 407,827chars_og/tok: 4.0encoded_chars/tok: 2.6
- id: _↵
· name: ���·つ ��_�� ���つ↵
· symbol: gib↵
- id:·000-capital↵
· name:·000 Capital↵
· symbol: '000'↵
- id:·01111010011110000110001001110100-token↵
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id:·01-token↵
· name: '01'↵
· symbol: '01'↵
- id:·0chain↵
· name: Zus↵
· symbol: zcn↵
- id:·0vix-protocol↵
· name:·0VIX Protocol↵
· symbol: vix↵
- id:·0x↵
· name:·0x Protocol↵
· symbol: zrx↵
- id:·0x0-ai-ai-smart-contract↵
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id:·0x678-landwolf-1933
... (truncated)
· name: ���·つ ��_�� ���つ↵
· symbol: gib↵
- id:·000-capital↵
· name:·000 Capital↵
· symbol: '000'↵
- id:·01111010011110000110001001110100-token↵
· name: '01111010011110000110001001110100'↵
· symbol: '01111010011110000110001001110100'↵
- id:·01-token↵
· name: '01'↵
· symbol: '01'↵
- id:·0chain↵
· name: Zus↵
· symbol: zcn↵
- id:·0vix-protocol↵
· name:·0VIX Protocol↵
· symbol: vix↵
- id:·0x↵
· name:·0x Protocol↵
· symbol: zrx↵
- id:·0x0-ai-ai-smart-contract↵
· name: '0x0.ai: AI Smart Contract'↵
· symbol: '0x0'↵
- id:·0x678-landwolf-1933
... (truncated)
TOON
chars: 623,383tokens: 243,750chars_og/tok: 6.6encoded_chars/tok: 2.6
[19332]{id,symbol,name}:↵
· _,gib,��·つ ��_�� ���つ↵
··000-capital,"000",000 Capital↵
··01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
··01-token,"01","01"↵
··0chain,zcn,Zus↵
··0vix-protocol,vix,0VIX Protocol↵
··0x,zrx,0x Protocol↵
··0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
··0x678-landwolf-1933,wolf,Landwolf↵
··0xgasless-2,0xgas,0xGasless↵
··0xgen,xgn,0xGen↵
··0x-leverage,oxl,0x Leverage↵
··0xlsd,0xlsd,0xLSD↵
··0xmonk,monk,0xMonk by Virtuals↵
··0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
··0xshadow,0xs,0xShadow↵
··0xsim-by-virtuals,sage,0xsim by Virtuals↵
··0xy,0xy,0xy↵
· "-10",loong,��↵
··1000bonk,1000bonk,1000BONK↵
··1000btt,1000btt,1000BTT↵
··1000cat,1000cat,1000CAT↵
··1000chems,1000cheems,1000CHEMS↵
··1000mog,1000mog,1000MOG
... (truncated)
· _,gib,��·つ ��_�� ���つ↵
··000-capital,"000",000 Capital↵
··01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"↵
··01-token,"01","01"↵
··0chain,zcn,Zus↵
··0vix-protocol,vix,0VIX Protocol↵
··0x,zrx,0x Protocol↵
··0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"↵
··0x678-landwolf-1933,wolf,Landwolf↵
··0xgasless-2,0xgas,0xGasless↵
··0xgen,xgn,0xGen↵
··0x-leverage,oxl,0x Leverage↵
··0xlsd,0xlsd,0xLSD↵
··0xmonk,monk,0xMonk by Virtuals↵
··0x-protocol-avalanche-bridged-zrx-e,zrx.e,Avalanche Bridged ZRX (Avalanche)↵
··0xshadow,0xs,0xShadow↵
··0xsim-by-virtuals,sage,0xsim by Virtuals↵
··0xy,0xy,0xy↵
· "-10",loong,��↵
··1000bonk,1000bonk,1000BONK↵
··1000btt,1000btt,1000BTT↵
··1000cat,1000cat,1000CAT↵
··1000chems,1000cheems,1000CHEMS↵
··1000mog,1000mog,1000MOG
... (truncated)
TSON
chars: 603,594tokens: 230,921chars_og/tok: 7.0encoded_chars/tok: 2.6
{@id,symbol,name#19332|_,gib,"��·つ ��_�� ���つ"|000-capital,"000","000 Capital"|01111010011110000110001001110100-token,"01111010011110000110001001110100","01111010011110000110001001110100"|01-token,"01","01"|0chain,zcn,Zus|0vix-protocol,vix,"0VIX Protocol"|0x,zrx,"0x Protocol"|0x0-ai-ai-smart-contract,0x0,"0x0.ai: AI Smart Contract"|0x678-landwolf-1933,wolf,Landwolf|0xgasless-2,0xgas,0xGasless|0xgen,xgn,0xGen|0x-leverage,oxl,"0x Leverage"|0xlsd,0xlsd,0xLSD|0xmonk,monk,"0xMonk by Virtuals"|0x-protocol-avalanche-bridged-zrx-e,zrx.e,"Avalanche Bridged ZRX (Avalanche)"|0xshadow,0xs,0xShadow|0xsim-by-virtuals,sage,"0xsim by Virtuals"|0xy,0xy,0xy|"-10",loong,��|1000bonk,1000bonk,1000BONK|1000btt,1000btt,1000BTT|1000cat,1000cat,1000CAT|1000chems,1000cheems,1000CHEMS|1000mog,1000mog,1000MOG|1000rats,1000rats,1000RATS|1000sats-ordinals,1000sats,"1000SATS (Ordinals)"|1000shib,1000shib,1000SHIB|1000x-by-virtuals,1000x,"1000x by Virtuals"|100-token,100¥,100¥|100xdarren,100x,100xDarren|10-figs,figs,"10 figs"|"-11",��长���,��长���|11am,11am,11am|1984-token,"1984","1984"|1art,1art,OneArt|1-coin-can-change-your-life,1-coin-can-change-your-life,"1 Coin Can Change Your Life"|1-community-can-change-your-life,community,"1 community can change your life"|1dev,1dev,1DEV|1-dog-can-change-your-life,1dog,"1 dog can change your life"|1-dollar-sol-coin,$1,$1|1guy,1guy,1GUY|1hive-water,water,"1Hive Water"|1hub-ai,1hub,1Hub.ai|1inch,1inch,1INCH|1inch-yvault,yv1inch,"1INCH yVault"|1intro,chef,CoinChef|1mbabydoge,1mbabydoge,1MBABYDOGE|1million-nfts,1mil,1MillionNFTs|1move-token,1mt,"1Move Token"|1-narrative-can-change-your-life,narrative,"1 narrative can change your life"|1-one,one,"1 (one)"|1-percent,1%,1%|1rus-btc25,"@btc25","@BTC25"|1rus-dao,1rusd,"1RUS DAO"|1-squirrel,peanut,"OG Peanut"|1-token,"1","1"|2004-pepe,bog,"2004 PEPE"|2025-token,"2025","2025 TOKEN"|2077-code,"2077","2077 CODE"|"2080","2080","2080"|21million,21m,21Million|23-turtles,ai23t,"23 Turtles"|2-3-years-and-forget,"23","2-3 years and forget"|24k-gold-pepe,goldpepe,"24K Gold PEPE"|2dai-io,2dai,"2DAI.io[Old]"|2dai-io-2,2dai,2DAI.io|2g-carbon-coin,2gcc,"2G Carbon Coin"|2moon,moon,2MOON|2-token,"2","2"|"-3",meow,"Meow Meow Coin"|3000-token,"3000","3000"|360noscope420blazeit,mlg,360noscope420blazeit|375ai,eat,375ai|39a-fun,39a,39a.fun|3a-lending-protocol,a3a,3A|3bubu,3bubu,3BuBu|3dpass,p3d,3DPass|3space-art,pace,"3SPACE ART"|"4",four,"4"|401jk,401jk,401jK|401k,401k,401K|404-gen,sn17,404—GEN|4-2,"4","4"|42069coin,42069coin,42069COIN|4-2-aminoethyl-benzene-1-2-diol,dopamine,"4-(2-Aminoethyl)benzene-1,2-diol"|42-coin,"42",42-coin|4444-token,"4444","4444"|4444-token-3,"4444","4444"|4547-token,"4547","4547"|47th-potus,trump47,"47th POTUS"|4chan,4chan,4Chan|4everland,4ever,4EVERLAND|4gentic,4gs,4GENTIC|4-next-unicorn,nxtu,"4 Next Unicorn"|4nonswap,4non,4nonSwap|4tb-coin,4tb,"4TB Coin"|4tool-ai,4tool,4TOOL.ai|4trump,4win,4TRUMP|4-way-mirror-money,4wmm,"4-Way Mirror Money"|"-5",������,������������������|500m-piece-of-paper,paper,"$500M piece of paper"|501-token,"501","501"|589-token,"589","589"|5ire,5ire,5ire|5mc,5mc,5mc|5tars,5tars,5TARS|5th-scape,$5scape,"5th Scape"|"-6"," "," "|666-token,"666","666"|67coin,"67",67COIN|"69420","69420","69420"|6chicken9,pop,6Chicken9|6ixrooms,6ixrooms,6ixROOMS|"-7",��,Voidify|717ai-by-virtuals,wire,"717ai by Virtuals"|777fuckilluminatiworldwid,fiw,777FuckIlluminatiWorldwid|"-8",���,���|8004-dog,dog8004,"8004 Dog"|8008-token,"8008","8008"|888coin,������,888Coin|888-token,"888","888"|88mph,mph,88mph|8-ball,sn125,"8 Ball"|8-bit-coin,coin,"8-Bit Coin"|8chan,8chan,8chan|8pay,8pay,8Pay|8-token,"8","8"|"-9",��,��|9-5,9-5,9to5|99-bitcoins,99btc,"99 Bitcoins"|99starz,stz,99Starz|9inch,9inch,9inch|9mm,9mm,9mm|9to5io,9to5,9to5io|a0x,a0x,A0x|a16gems,a16g,a16gems|a16z-ai-dog,tilly,"a16z AI Dog"|a51-finance,a51,"A51 Finance"|a7a5,a7a5,A7A5|aaa-cat,aaa,"aaa cat"|aaai_agent-by-virtuals,aaai,"AAAI_agent by Virtuals"|aada-finance,lenfi,Lenfi|aadex-finance,ade,"AADex Finance"|aagent-ai,aai,Aagent.ai|aag-ventures,aag,AAG|aardvark-2,vark,Aardvark|aark-digital,aark,"Aark Digital"|aarna-afi-802v2,"afi·802v2","aarna afi·802v2"|aarna-atv111,atv111,"aarna atv111"|aarna-atv111-arbitrum,atv111,"aarna atv111 (Arbitrum)"|aarna-atv111-sonic,atv111,"aarna atv111 (Sonic)"|aarna-atv-808,atv808,"aarna atv·808"|aarna-atv-usdc,atvusdc,"aarna atv USDC (Arbitrum)"|aarna-atv-usdc-ethereum,atvusdc,"aarna atv USDC (Ethereum)"|aastoken,aast,AASToken|aave,aave,Aave|aave-aave,aaave,"Aave AAVE"|aave-amm-bptbalweth,aammbptbalweth,"Aave AMM BptBALWETH"|aave-amm-bptwbtcweth,aammbptwbtcweth,"Aave AMM BptWBTCWETH"|aave-amm-dai,aammdai,"Aave AMM DAI"|aave-amm-uniaaveweth,aammuniaaveweth,"Aave AMM UniAAVEWETH"|aave-amm-unibatweth,aammunibatweth,"Aave AMM UniBATWETH"|aave-amm-unicrvweth,aammunicrvweth,"Aave AMM UniCRVWETH"|aave-amm-unidaiusdc,aammunidaiusdc,"Aave AMM UniDAIUSDC"|aave-amm-unidaiweth,aammunidaiweth,"Aave AMM UniDAIWETH"|aave-amm-unilinkweth,aammunilinkweth,"Aave AMM UniLINKWETH"|aave-amm-unimkrweth,aammunimkrweth,"Aave AMM·
... (truncated)
... (truncated)
minemizer
chars: 622,925tokens: 236,563chars_og/tok: 6.8encoded_chars/tok: 2.6
id; symbol; name↵
_; gib; ���·つ ��_�� ���つ↵
000-capital;·000;·000 Capital↵
01111010011110000110001001110100-token;·01111010011110000110001001110100;·01111010011110000110001001110100↵
01-token;·01;·01↵
0chain; zcn; Zus↵
0vix-protocol; vix;·0VIX Protocol↵
0x; zrx;·0x Protocol↵
0x0-ai-ai-smart-contract;·0x0;·0x0.ai: AI Smart Contract↵
0x678-landwolf-1933; wolf; Landwolf↵
0xgasless-2;·0xgas;·0xGasless↵
0xgen; xgn;·0xGen↵
0x-leverage; oxl;·0x Leverage↵
0xlsd;·0xlsd;·0xLSD↵
0xmonk; monk;·0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow;·0xs;·0xShadow↵
0xsim-by-virtuals; sage;·0xsim by Virtuals↵
0xy;·0xy;·0xy↵
-10; loong; ���↵
1000bonk;·1000bonk;·1000BONK↵
1000btt;·1000btt;·1000BTT↵
1000cat;·1000cat;·1000CAT↵
1000chems;·1000cheems;·1000CHEMS↵
1000mog;·1000mog;·1000MOG
... (truncated)
_; gib; ���·つ ��_�� ���つ↵
000-capital;·000;·000 Capital↵
01111010011110000110001001110100-token;·01111010011110000110001001110100;·01111010011110000110001001110100↵
01-token;·01;·01↵
0chain; zcn; Zus↵
0vix-protocol; vix;·0VIX Protocol↵
0x; zrx;·0x Protocol↵
0x0-ai-ai-smart-contract;·0x0;·0x0.ai: AI Smart Contract↵
0x678-landwolf-1933; wolf; Landwolf↵
0xgasless-2;·0xgas;·0xGasless↵
0xgen; xgn;·0xGen↵
0x-leverage; oxl;·0x Leverage↵
0xlsd;·0xlsd;·0xLSD↵
0xmonk; monk;·0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e; zrx.e; Avalanche Bridged ZRX (Avalanche)↵
0xshadow;·0xs;·0xShadow↵
0xsim-by-virtuals; sage;·0xsim by Virtuals↵
0xy;·0xy;·0xy↵
-10; loong; ���↵
1000bonk;·1000bonk;·1000BONK↵
1000btt;·1000btt;·1000BTT↵
1000cat;·1000cat;·1000CAT↵
1000chems;·1000cheems;·1000CHEMS↵
1000mog;·1000mog;·1000MOG
... (truncated)
minemizer (compact)
chars: 584,259tokens: 236,253chars_og/tok: 6.8encoded_chars/tok: 2.5
id;symbol;name↵
_;gib;��·つ ��_�� ���つ↵
000-capital;000;000 Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIX Protocol↵
0x;zrx;0x Protocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0x Leverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsim by Virtuals↵
0xy;0xy;0xy↵
-10;loong;��↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
_;gib;��·つ ��_�� ���つ↵
000-capital;000;000 Capital↵
01111010011110000110001001110100-token;01111010011110000110001001110100;01111010011110000110001001110100↵
01-token;01;01↵
0chain;zcn;Zus↵
0vix-protocol;vix;0VIX Protocol↵
0x;zrx;0x Protocol↵
0x0-ai-ai-smart-contract;0x0;0x0.ai: AI Smart Contract↵
0x678-landwolf-1933;wolf;Landwolf↵
0xgasless-2;0xgas;0xGasless↵
0xgen;xgn;0xGen↵
0x-leverage;oxl;0x Leverage↵
0xlsd;0xlsd;0xLSD↵
0xmonk;monk;0xMonk by Virtuals↵
0x-protocol-avalanche-bridged-zrx-e;zrx.e;Avalanche Bridged ZRX (Avalanche)↵
0xshadow;0xs;0xShadow↵
0xsim-by-virtuals;sage;0xsim by Virtuals↵
0xy;0xy;0xy↵
-10;loong;��↵
1000bonk;1000bonk;1000BONK↵
1000btt;1000btt;1000BTT↵
1000cat;1000cat;1000CAT↵
1000chems;1000cheems;1000CHEMS↵
1000mog;1000mog;1000MOG
... (truncated)
complex_mixed.json — Original size: 1320 chars — Tokenizer: phi4 (microsoft/phi-4)
JSON (pretty)
chars: 1,320tokens: 427chars_og/tok: 3.1encoded_chars/tok: 3.1
[↵
· {↵
··· "id":·1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id":·2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
· {↵
··· "id":·1,↵
··· "profile": {↵
····· "name": "Grace",↵
····· "location": {↵
······· "city": "NYC",↵
······· "country": "USA"↵
····· }↵
··· },↵
··· "tags": [↵
····· "admin",↵
····· "verified"↵
··· ],↵
··· "metadata": {↵
····· "created": "2024-01-15"↵
··· }↵
· },↵
· {↵
··· "id":·2,↵
··· "profile": {↵
····· "name": "Henry",↵
····· "location": {↵
······· "city": "London",↵
······· "country": "UK"
... (truncated)
JSON (min)
chars: 760tokens: 218chars_og/tok: 6.1encoded_chars/tok: 3.5
[{"id":1,"profile":{"name":"Grace","location":{"city":"NYC","country":"USA"}},"tags":["admin","verified"],"metadata":{"created":"2024-01-15"}},{"id":2,"profile":{"name":"Henry","location":{"city":"London","country":"UK"}},"tags":["user"],"metadata":{"created":"2024-02-20","updated":"2024-03-10"}},{"id":3,"profile":{"name":"Ivy","location":{"city":"Tokyo","country":"Japan"}},"tags":["moderator","verified","premium"],"metadata":{"created":"2024-01-05"}},{"id":4,"profile":{"name":"Jack","location":{"city":"Sydney","country":"Australia"}},"tags":["user","new"],"metadata":{"created":"2024-04-01"}},{"id":5,"profile":{"name":"Kate","location":{"city":"Berlin","country":"Germany"}},"tags":["admin"],"metadata":{"created":"2023-12-01","updated":"2024-02-15"}}]
CSV: N/A
TSV: N/A
YAML
chars: 818tokens: 278chars_og/tok: 4.7encoded_chars/tok: 2.9
- id:·1↵
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC↵
····· country: USA↵
··· name: Grace↵
· tags:↵
· - admin↵
· - verified↵
- id:·2↵
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London↵
····· country: UK↵
··· name: Henry↵
· tags:↵
· - user↵
- id:·3↵
· metadata:↵
··· created: '2024-01-05'
... (truncated)
· metadata:↵
··· created: '2024-01-15'↵
· profile:↵
··· location:↵
····· city: NYC↵
····· country: USA↵
··· name: Grace↵
· tags:↵
· - admin↵
· - verified↵
- id:·2↵
· metadata:↵
··· created: '2024-02-20'↵
··· updated: '2024-03-10'↵
· profile:↵
··· location:↵
····· city: London↵
····· country: UK↵
··· name: Henry↵
· tags:↵
· - user↵
- id:·3↵
· metadata:↵
··· created: '2024-01-05'
... (truncated)
TOON
chars: 881tokens: 276chars_og/tok: 4.8encoded_chars/tok: 3.2
[5]:↵
· - id:·1↵
··· profile:↵
····· name: Grace↵
····· location:↵
······· city: NYC↵
······· country: USA↵
··· tags[2]: admin,verified↵
··· metadata:↵
····· created:·2024-01-15↵
· - id:·2↵
··· profile:↵
····· name: Henry↵
····· location:↵
······· city: London↵
······· country: UK↵
··· tags[1]: user↵
··· metadata:↵
····· created:·2024-02-20↵
····· updated:·2024-03-10↵
· - id:·3↵
··· profile:↵
····· name: Ivy↵
····· location:↵
······· city: Tokyo
... (truncated)
· - id:·1↵
··· profile:↵
····· name: Grace↵
····· location:↵
······· city: NYC↵
······· country: USA↵
··· tags[2]: admin,verified↵
··· metadata:↵
····· created:·2024-01-15↵
· - id:·2↵
··· profile:↵
····· name: Henry↵
····· location:↵
······· city: London↵
······· country: UK↵
··· tags[1]: user↵
··· metadata:↵
····· created:·2024-02-20↵
····· updated:·2024-03-10↵
· - id:·3↵
··· profile:↵
····· name: Ivy↵
····· location:↵
······· city: Tokyo
... (truncated)
TSON
chars: 453tokens: 175chars_og/tok: 7.5encoded_chars/tok: 2.6
{@id,profile(@name,location),tags,metadata#5|1,{Grace,{@city,country|NYC,USA}},[admin,verified],{@created|2024-01-15}|2,{Henry,{@city,country|London,UK}},[user],{@created,updated|2024-02-20,2024-03-10}|3,{Ivy,{@city,country|Tokyo,Japan}},[moderator,verified,premium],{@created|2024-01-05}|4,{Jack,{@city,country|Sydney,Australia}},[user,new],{@created|2024-04-01}|5,{Kate,{@city,country|Berlin,Germany}},[admin],{@created,updated|2023-12-01,2024-02-15}}
minemizer
chars: 421tokens: 163chars_og/tok: 8.1encoded_chars/tok: 2.6
id; profile{ name; location{ city; country}}; tags[]; metadata{ created; ...}↵
1; { Grace; { NYC; USA}}; [ admin; verified]; {·2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; {·2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; {·2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; {·2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; {·2023-12-01; updated:2024-02-15}
1; { Grace; { NYC; USA}}; [ admin; verified]; {·2024-01-15}↵
2; { Henry; { London; UK}}; [ user]; {·2024-02-20; updated:2024-03-10}↵
3; { Ivy; { Tokyo; Japan}}; [ moderator; verified; premium]; {·2024-01-05}↵
4; { Jack; { Sydney; Australia}}; [ user; new]; {·2024-04-01}↵
5; { Kate; { Berlin; Germany}}; [ admin]; {·2023-12-01; updated:2024-02-15}
minemizer (compact)
chars: 364tokens: 163chars_og/tok: 8.1encoded_chars/tok: 2.2
id;profile{ name;location{ city;country}};tags[];metadata{ created;...}↵
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}
1;{Grace;{NYC;USA}};[admin;verified];{2024-01-15}↵
2;{Henry;{London;UK}};[user];{2024-02-20;updated:2024-03-10}↵
3;{Ivy;{Tokyo;Japan}};[moderator;verified;premium];{2024-01-05}↵
4;{Jack;{Sydney;Australia}};[user;new];{2024-04-01}↵
5;{Kate;{Berlin;Germany}};[admin];{2023-12-01;updated:2024-02-15}